Better Metrics Through Scripting Matt Williams Evangelist @ Datadog Organizer @ DevOpsDays Boston 2017 @technovangelist http://dtdg.co/nginx17deck 

@technovangelist 

@technovangelist Datadog is a metrics company 

@technovangelist We collect via an agent 

@technovangelist Datadog Agent nginx integration postgres integration fluentd integration Start the Agent Every 15 seconds run the integrations 

@technovangelist The simplest agent config for nginx init_config: instances: - nginx_status_url: http://localhost/nginx_status/ 

@technovangelist nginx_status NGINX NGINX Plus stub_status_module http_status_module location /nginx_status { stub_status; } server { location / { health_check; } status_zone server_backend; location /nginx_status { status; } } 

@technovangelist {"version":8,"nginx_version":"1.11.10","nginx_build":"nginx-plus-r12-p3","address":"206.251.255.64","generation":5,"load_timestamp": 1501322401673,"timestamp":1501861405424,"pid":26474,"ppid":96917,"processes":{"respawned":0},"connections":{"accepted":23164115,"dropp 752731,"active":9,"idle":386},"ssl":{"handshakes":255730,"handshakes_failed":41151,"session_reuses":35906},"requests":{"total": 48895013,"current":7},"server_zones":{"hg.nginx.org":{"processing":1,"requests":1310186,"responses":{"1xx":0,"2xx":1293954,"3xx":12629 2419,"5xx":993,"total":1309995},"discarded":190,"received":325707828,"sent":38155544995},"trac.nginx.org":{"processing":5,"requests": 445206,"responses":{"1xx":0,"2xx":273454,"3xx":91942,"4xx":7008,"5xx":21013,"total":393417},"discarded":51784,"received":123051863,"se 11514166530},"lxr.nginx.org":{"processing":0,"requests":8488,"responses":{"1xx":0,"2xx":6743,"3xx":406,"4xx":556,"5xx":490,"total": 8195},"discarded":293,"received":3342520,"sent":280101100}},"slabs":{"bot":{"pages":{"used":2,"free":29},"slots":{"8":{"used":0,"free" 0,"reqs":0,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"32":{"used":1,"free":126,"reqs":1,"fails":0},"64":{"used":0,"free": 0,"reqs":0,"fails":0},"128":{"used":10,"free":22,"reqs":375,"fails":0},"256":{"used":0,"free":0,"reqs":0,"fails":0},"512":{"used":0,"f 0,"reqs":0,"fails":0},"1024":{"used":0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"free":0,"reqs":0,"fails":0}}},"ban":{"pages":{"u 2,"free":29},"slots":{"8":{"used":0,"free":0,"reqs":0,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"32":{"used":1,"free": 126,"reqs":1,"fails":0},"64":{"used":0,"free":0,"reqs":0,"fails":0},"128":{"used":1,"free":31,"reqs":1,"fails":0},"256":{"used":0,"fre 0,"reqs":0,"fails":0},"512":{"used":0,"free":0,"reqs":0,"fails":0},"1024":{"used":0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"fre 0,"reqs":0,"fails":0}}},"one":{"pages":{"used":3,"free":251},"slots":{"8":{"used":0,"free":0,"reqs":0,"fails":0},"16":{"used":0,"free" 0,"reqs":0,"fails":0},"32":{"used":1,"free":126,"reqs":1,"fails":0},"64":{"used":0,"free":0,"reqs":0,"fails":0},"128":{"used":17,"free 47,"reqs":407166,"fails":0},"256":{"used":0,"free":0,"reqs":0,"fails":0},"512":{"used":0,"free":0,"reqs":0,"fails":0},"1024":{"used": 0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"free":0,"reqs":0,"fails":0}}},"http_cache":{"pages":{"used":1024,"free":3047},"slots" {"used":0,"free":0,"reqs":0,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"32":{"used":0,"free":0,"reqs":0,"fails":0},"64":{" 1,"free":63,"reqs":1,"fails":0},"128":{"used":0,"free":0,"reqs":0,"fails":0},"256":{"used":16284,"free":68,"reqs":6557387,"fails":0}," {"used":1,"free":7,"reqs":1,"fails":0},"1024":{"used":0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"free":0,"reqs":0,"fails":0}}}," {"pages":{"used":1175,"free":11550},"slots":{"8":{"used":0,"free":0,"reqs":0,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"3 {"used":0,"free":0,"reqs":0,"fails":0},"64":{"used":1,"free":63,"reqs":1,"fails":0},"128":{"used":7260,"free":5412,"reqs":181010,"fail 0},"256":{"used":7259,"free":5189,"reqs":180672,"fails":0},"512":{"used":0,"free":0,"reqs":0,"fails":0},"1024":{"used":0,"free":0,"req 4,"fails":0},"2048":{"used":0,"free":0,"reqs":332,"fails":0}}},"trac-backend":{"pages":{"used":5,"free":2},"slots":{"8":{"used":1,"fre 503,"reqs":1,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"32":{"used":4,"free":123,"reqs":4,"fails":0},"64":{"used":1,"free 63,"reqs":1,"fails":0},"128":{"used":6,"free":26,"reqs":6,"fails":0},"256":{"used":0,"free":0,"reqs":0,"fails":0},"512":{"used":2,"fre 6,"reqs":2,"fails":0},"1024":{"used":0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"free":0,"reqs":0,"fails":0}}},"hg-backend":{"pag {"used":4,"free":3},"slots":{"8":{"used":1,"free":503,"reqs":1,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"32":{"used":5," 122,"reqs":5,"fails":0},"64":{"used":0,"free":0,"reqs":0,"fails":0},"128":{"used":6,"free":26,"reqs":6,"fails":0},"256":{"used":0,"fre 0,"reqs":0,"fails":0},"512":{"used":2,"free":6,"reqs":2,"fails":0},"1024":{"used":0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"fre 0,"reqs":0,"fails":0}}},"lxr-backend":{"pages":{"used":4,"free":3},"slots":{"8":{"used":1,"free":503,"reqs":1,"fails":0},"16":{"used": 0,"free":0,"reqs":0,"fails":0},"32":{"used":5,"free":122,"reqs":5,"fails":0},"64":{"used":0,"free":0,"reqs":0,"fails":0},"128":{"used" 6,"free":26,"reqs":6,"fails":0},"256":{"used":0,"free":0,"reqs":0,"fails":0},"512":{"used":2,"free":6,"reqs":2,"fails":0},"1024":{"use 0,"free":0,"reqs":0,"fails":0},"2048":{"used":0,"free":0,"reqs":0,"fails":0}}},"demo-backend":{"pages":{"used":5,"free":2},"slots":{"8 {"used":1,"free":503,"reqs":1,"fails":0},"16":{"used":0,"free":0,"reqs":0,"fails":0},"32":{"used":2,"free":125,"reqs":2,"fails":0},"64 {"used":1,"free":63,"reqs":1,"fails":0},"128":{"used":4,"free":28,"reqs":4,"fails":0},"256":{"used":0,"free":0,"reqs":0,"fails":0},"51 

@technovangelist Active connections: 4 server accepts handled requests 5 5 4 Reading: 0 Writing: 1 Waiting: 3 

@technovangelist NGINX metrics nginx.net.writing nginx.net.waiting nginx.net.reading nginx.net.connections nginx.net.request_per_s nginx.net.conn_opened_per_s nginx.net.conn_dropped_per_s 

@technovangelist NGINX Plus metrics nginx.net.writing nginx.net.waiting nginx.net.reading nginx.connections.accepted nginx.connections.active nginx.connections.dropped nginx.connections.idle nginx.generation nginx.load_timestamp nginx.pid nginx.processes.respawned nginx.requests.current nginx.requests.total nginx.server_zone.discarded nginx.server_zone.processing nginx.server_zone.received nginx.server_zone.requests nginx.server_zone.responses.1xx nginx.server_zone.responses.2xx nginx.server_zone.responses.3xx nginx.server_zone.responses.4xx nginx.server_zone.responses.5xx nginx.server_zone.responses.total nginx.server_zone.sent nginx.ssl.handshakes nginx.ssl.handshakes_failed nginx.ssl.session_reuses nginx.timestamp nginx.upstream.keepalive nginx.upstream.peers.active nginx.upstream.peers.backup nginx.upstream.peers.downstart nginx.upstream.peers.downtime nginx.upstream.peers.fails nginx.upstream.peers.id nginx.upstream.peers.received nginx.upstream.peers.requests nginx.upstream.peers.responses. nginx.upstream.peers.selected nginx.upstream.peers.sent nginx.upstream.peers.weight nginx.version 

@technovangelist How to get more metrics •Watch an aggregated stats view • nginx_status • Create your own •Let your log aggregation tool create stats • Configure your log output •Send each request’s stats 

@technovangelist $ancient_browser $arg_ $args $binary_remote_addr $body_bytes_sent $bytes_received $bytes_sent $connection $connection_requests $connections_active $connections_reading $connections_waiting $connections_writing $content_length $content_type $cookie_ $date_gmt $date_local $geoip_region $geoip_region_name $gzip_ratio $host $hostname $http2 $http_ $https $invalid_referer $is_args $jwt_claim_ $jwt_header_ $limit_rate $memcached_key $modern_browser $msec $msie $nginx_version $request_body $request_body_file $request_completion $request_filename $request_id $request_length $request_method $request_time $request_uri $scheme $secure_link $secure_link_expires $sent_http_ $sent_trailer_ $server_addr $server_name $server_port $server_protocol $ssl_client_v_end $ssl_client_v_remain $ssl_client_v_start $ssl_client_verify $ssl_curves $ssl_preread_server_nam $ssl_protocol $ssl_server_name $ssl_session_id $ssl_session_reused $status $tcpinfo_rtt $tcpinfo_rttvar $tcpinfo_snd_cwnd $tcpinfo_rcv_space $time_iso8601 $time_local $uid_got Available NGINX Variables 

@technovangelist Sample Log Config http { log_format compression '$remote_addr - $remote_user [$time_local] ' '"$request" $status $body_bytes_sent ' '"$http_referer" "$http_user_agent" "$gzip_ratio"'; server { gzip on; access_log /spool/logs/nginx-access.log compression; ... } } 

@technovangelist What about scripting? 

@technovangelist Scripting Choices •lua - OpenResty • Easy to learn lua (http://dtdg.co/nginx17-lua) • More mature •nginScript • You may already know Javascript • Created by NGINX 

@technovangelist Scripting Benefits •Per-request config file • Different log file config per: • Set of pages • Set of users •Provide additional functionality • Sub-requests • Reloader - http://dtdg.co/nginx17-reload 

@technovangelist server { listen 1337; location /reload { content_by_lua ‘os.execute("/usr/sbin/ nginx -s reload")'; } } http://dtdg.co/nginx17-reload 

@technovangelist server { listen 1337; location /reload { content_by_lua ‘os.execute("/usr/sbin/ nginx -s reload")'; } } 

@technovangelist server { listen 1337; location /reload { content_by_lua ‘os.execute("/usr/sbin/ nginx -s reload")'; } } 

@technovangelist Scripting Gotchas •Scripting environment flushed per request •Lua can get around this • Buckets • Sub requests •nginScript maybe later this year 

@technovangelist •Watch an aggregated stats view •Let your log aggregation tool create stats •Send each request’s stats 

@technovangelist •Watch an aggregated stats view •Let your log aggregation tool create stats •Send each request’s stats 

@technovangelist Aggregated Stats - luameter.com 

@technovangelist Aggregated Stats - luameter.com { "connections": {"active": "25","reading": "0","waiting": "24","writing": "1"}, "luameter_version": "1.0", "meta_version": "97fe7f", "nginx_version": "1.7.9", "ngx_lua_version": "0.9.13", "timestamp": 1502896390, "upstreams": { "127.0.0.1:8080": { "cache": {"bypass": 0,"expired": 0,"hit": 0,"miss": 0,"revalidated": 0,"stal "latency": {"max": 0,"mean": 0,"median": 0,"min": 0,"p75": 0,"p90": 0,"p95": "rates": {"fifteen_minute_avg": 4.45e-322,"five_minute_avg": 1.5e-322,"mean" "received": 23281719, "requests": 23281719, "responses": {"200": 23281718,"1xx": 0,"2xx": 23281718,"3xx": 0,"4xx": 0,"5x "sent": 46563436 }, "127.0.0.1:8081": { "cache": {"bypass": 0,"expired": 0,"hit": 0,"miss": 0,"revalidated": 0,"stal "latency": {"max": 0,"mean": 0,"median": 0,"min": 0,"p75": 0,"p90": 0,"p95": 

@technovangelist lua-resty-stats http://dtdg.co/nginx17-restystat 

@technovangelist •Watch an aggregated stats view •Let your log aggregation tool create stats •Send each request’s stats 

@technovangelist Custom Log File with nginScript js_include conf.d/header_logging.js; js_set $access_log_with_headers kvAccessLog; log_format kvpairs $access_log_with_headers; server { listen 80; access_log /var/log/nginx/access_headers.log kvpairs; location / { proxy_pass http://www.example.com; } } https://www.nginx.com/blog/introduction-nginscript/ 

@technovangelist Custom Log File with nginScript js_include conf.d/header_logging.js; js_set $access_log_with_headers kvAccessLog; log_format kvpairs $access_log_with_headers; server { listen 80; access_log /var/log/nginx/access_headers.log kvpairs; location / { proxy_pass http://www.example.com; } } https://www.nginx.com/blog/introduction-nginscript/ 

@technovangelist Custom Log File with nginScript js_include conf.d/header_logging.js; js_set $access_log_with_headers kvAccessLog; log_format kvpairs $access_log_with_headers; server { listen 80; access_log /var/log/nginx/access_headers.log kvpairs; location / { proxy_pass http://www.example.com; } } https://www.nginx.com/blog/introduction-nginscript/ 

@technovangelist function kvAccessLog(req, res) { var log = req.variables.time_iso8601; log += " client=" + req.remoteAddress; log += " method=" + req.method; log += " uri=" + req.uri; log += " status=" + res.status; log += kvHeaders(req.headers, "req"); log += kvHeaders(res.headers, "res"); return log; } https://www.nginx.com/blog/introduction-nginscript/ 

@technovangelist function kvAccessLog(req, res) { var log = req.variables.time_iso8601; log += " client=" + req.remoteAddress; log += " method=" + req.method; log += " uri=" + req.uri; log += " status=" + res.status; log += kvHeaders(req.headers, "req"); log += kvHeaders(res.headers, "res"); return log; } https://www.nginx.com/blog/introduction-nginscript/ 

@technovangelist function kvHeaders(headers, parent) { var kvpairs = ""; for (var h in headers) { kvpairs += " " + parent + "." + h + "="; if ( headers[h].indexOf(" ") == -1 ) { kvpairs += headers[h]; } else { kvpairs += "'" + headers[h] + "'"; } } return kvpairs; } https://www.nginx.com/blog/introduction-nginscript/ 

@technovangelist Custom Log File with nginScript js_include conf.d/header_logging.js; js_set $access_log_with_headers kvAccessLog; log_format kvpairs $access_log_with_headers; server { listen 80; access_log /var/log/nginx/access_headers.log kvpairs; location / { proxy_pass http://www.example.com; } } 

@technovangelist Custom Log File with nginScript js_include conf.d/header_logging.js; js_set $access_log_with_headers kvAccessLog; log_format kvpairs $access_log_with_headers; server { listen 80; access_log /var/log/nginx/access_headers.log kvpairs; location / { proxy_pass http://www.example.com; } } 

@technovangelist •Watch an aggregated stats view •Let your log aggregation tool create stats •Send each request’s stats 

@technovangelist ngx_lua_datadog location { local conf = { host = "127.0.0.1", port = 8125, namespace = "Stats", timeout = 1} local statsd_logger = require "ngx_lua_datadog" local logger, err = statsd_logger:new(conf) if err then ngx_log(ngx.ERR, "failed to create Statsd logger: ", err) end if ( my_condition ) then logger:counter("condition_matched", 1, 1) ngx.exit(204) else logger:counter("condition_not_matched", 1, 1) ngx.exit(200) end } http://dtdg.co/nginx17-luadatadog 

@technovangelist ngx_lua_datadog location { local conf = { host = "127.0.0.1", port = 8125, namespace = "Stats", timeout = 1} local statsd_logger = require "ngx_lua_datadog" local logger, err = statsd_logger:new(conf) if err then ngx_log(ngx.ERR, "failed to create Statsd logger: ", err) end if ( my_condition ) then logger:counter("condition_matched", 1, 1) ngx.exit(204) else logger:counter("condition_not_matched", 1, 1) ngx.exit(200) end } http://dtdg.co/nginx17-luadatadog 

@technovangelist ngx_lua_datadog location { local conf = { host = "127.0.0.1", port = 8125, namespace = "Stats", timeout = 1} local statsd_logger = require "ngx_lua_datadog" local logger, err = statsd_logger:new(conf) if err then ngx_log(ngx.ERR, "failed to create Statsd logger: ", err) end if ( my_condition ) then logger:counter("condition_matched", 1, 1) ngx.exit(204) else logger:counter("condition_not_matched", 1, 1) ngx.exit(200) end } http://dtdg.co/nginx17-luadatadog 

@technovangelist ngx_lua_datadog function new(conf) local sock = ngx.socket.udp() sock:settimeout(conf.timeout) local _, err = sock:setpeername(conf.host, conf.port) if err then return nil, "failed to connect to "..conf.host..":"..conf.port..": "..err end local dogstatsd = { host = conf.host, port = conf.port, socket = sock, namespace = conf.namespace } return setmetatable(dogstatsd, dogstatsd_mt) end http://dtdg.co/nginx17-luadatadog 

@technovangelist ngx_lua_datadog location { local conf = { host = "127.0.0.1", port = 8125, namespace = "Stats", timeout = 1} local statsd_logger = require "ngx_lua_datadog" local logger, err = statsd_logger:new(conf) if err then ngx_log(ngx.ERR, "failed to create Statsd logger: ", err) end if ( my_condition ) then logger:counter("condition_matched", 1, 1) ngx.exit(204) else logger:counter("condition_not_matched", 1, 1) ngx.exit(200) end } http://dtdg.co/nginx17-luadatadog 

@technovangelist ngx_lua_datadog function counter(stat, value, sample_rate, tags) return self:send_dogstatsd(stat, value, "c", sample_rate, tags) end function send_dogstatsd(stat, delta, kind, sample_rate, tags) local udp_message = self:create_dogstatsd_message(stat, delta, kind, sample_rate, tags) local ok, err = self.socket:send(udp_message) if not ok then ngx.log(ngx.ERR, "failed to send data to ………………, err) end end http://dtdg.co/nginx17-luadatadog 

@technovangelist Use nginx-dogstatsd server { listen 80; server_name www.your.domain.com; dogstatsd_count "your_product.requests" 1; location / { dogstatsd_count "your_product.pages.index_requests" 1; dogstatsd_count “your_prod…ex_requests” 1 "server:$server_name"; dogstatsd_count “your_prod…ex_responses” 1 "" "$request_completion"; dogstatsd_timing “your_prod…ex_response_time” "$upstream_response_time"; dogstatsd_count “your_prod…om_$upstream_http_x_some_custom_header” 1 "" "$upstream_http_x_some_custom_header"; proxy_pass http://some.other.domain.com; } } http://dtdg.co/nginx17-dogstatsd 

@technovangelist Use nginx-dogstatsd / nginScript js_include conf.d/metrics.js; js_set $us_requests usrequests; js_set $uk_requests ukrequests; js_set $dnt_requests dntrequests; js_set $process_time proctime; server { listen 80; dogstatsd_count "product.requests.usa" $us_requests; dogstatsd_count "product.requests.uk" $uk_requests; dogstatds__count "do_not_track.requests" $dnt_requests; dogstatsd_timing "product.requests.timing" $process_time; ... } http://dtdg.co/nginx17-dogstatsd 

@technovangelist Review •Scripting makes it possible to get better metrics •Can use Lua or nginScript •Lua is more functional today, nginScript has potential 

@technovangelist Other cool things to do Dev Essentials lua-resty-cookie Routing Libraries Middleware Kong Templating Liquid Validation Authentication Cryptography Auto SSL with Let’s Encrypt Network Database / Storage mysql, postgres, mongo Profiling Flame Graph MQ Kafka Bar/QR Code Compression Image Formats Caching Logging Awesome Resty - http://dtdg.co/nginx17-awesome 

@technovangelist Finding more information The Latest and Greatest from ngx_lua: New Features & Tools http://dtdg.co/ddnginx-vid1 A New and Powerful Way to Configure NGINX, nginScript
 http://dtdg.co/ddnginx-vid3 

Better Metrics Through Scripting Matt Williams Evangelist @ Datadog Organizer @ DevOpsDays Boston 2017 @technovangelist [email protected]