Engineers

but...

We're building complex systems

so...

How do you know what to optimize?

How do you know if you've been successful?

Metrics

allow you to explore

allow you to determine what's important

allow you to focus on what's important

enable educated decisions

Graphite

Stores numeric time-series data
Renders graphs of this data on demand
Exports data via REST API

Graphite Architecture

carbon - daemon that listens for time-series data
whisper - database for storing data
webapp - webapp that renders graphs

Data retention

How often can you produce your data?
What is the finest precision you will require?
How long will you need that level of precision?
What is the coarsest precision you can use?
How far back would you ever need to see data?

StatsD

NodeJS UDP network daemon
Listens for stats and aggregates before sending to graphite
Provides support for different types of metrics

NodeJS StatsD Clients

Getting started with Lynx

Install

npm install lynx

example1.js:

var lynx = require('lynx');
var opt = {}; // opt.prefix = 'myPrefix';
var metrics = new lynx('162.242.212.227', 8125, opt); // StatsD IP & Port
metrics.gauge('example1.gauge', 300); // Send our first metric

Metric Types

Gauge
Counter
Timer
Set

Gauges

// example2.js
function updateGauges() {
  metrics.gauge('example2.gauge1', Math.random()+3);
  metrics.gauge('example2.gauge2', Math.random());
  setTimeout(updateGauges, 500);
}
updateGauges();

Counters

//example3.js
function increment() {
  for (var j=10; j‹Math.random()*40; j++) {
    metrics.increment('example3.counter2');
    // metrics.decrement('example3.counter2');
  }
  setTimeout(increment, 500);
}
increment();

Timers

//example4.js
function timer() {
  for (var i=0; i‹20; i++) {
    metrics.timing('example4.timer1', Math.random()*50); // time in ms
  }
  setTimeout(timer, 500);
}
timer();

//example5.js - Timers Redux
function timer() {
  var timer = metrics.createTimer('example5.timer1');
  setTimeout(function(){timer.stop()}, Math.random()*25);
}
function timers() {
  for (var i=0; i‹20; i++) {
    timer();
  }
  setTimeout(timers, 500);
}
timers();

Sets

// example6.js
function pick() {
  metrics.set('example6.user', 'ross');
  metrics.set('example6.user', 'ross');
  if (Math.random() > 0.9) { metrics.set('example6.user', 'jared'); }
  setTimeout(pick, 1000);
}
pick();

Sampling

'really, really frequent' data can overwhelm StatsD


  metrics.increment('some.counter', 0.1); // Send only 10% of calls

lynx-express

Install

npm install lynx-express

Use Middleware


// Import lynx and lynx-express
var lynx = require('lynx');
var lynxExpress = require('lynx-express');

// Setup your lynx StatsD client as normal
var metrics = new lynx('162.242.212.227', 8125, {prefix: 'express'});

// Create the lynx Express middleware
var lynxMiddleware = lynxExpress(metrics);

// Tell Express to use your lynx middleware
server.use(lynxMiddleware());
// or server.use(lynxMiddleware({timeByUrl: true});

Express Server

// Express server
var express = require('express');
var app = express();
setupLynxMiddleware(app);
function randResp(req, res, next) {
  var delay = 1000;
  if (Math.random() > 0.95) {
    delay = delay*2;
  }
  setTimeout(function() {res.send(req.url);}, Math.random()*delay);
};
app.get('/', randResp);
app.get('/users/', randResp);
app.get('/users/:user', randResp);
app.listen(3000);

Timing For Each Route

Express Route Timing

Graphite demo

Configuration changes

# Changes to /opt/graphite/webapp/content/js/composer_widgets.js
DEFAULT_CACHE_DURATION = 1 # Cache images and data for 1 minute

// Changes to /opt/graphite/webapp/content/js/composer_widgets.js
var doRefresh = function () {
  Composer.updateImage();
  var interval = 1; // Change refresh rate for widgets to every second                                                                                  
  button.timer = setTimeout(doRefresh, interval * 1000)
}

// Changes to /etc/statsd/config.js
"flushInterval": 1000,

Thanks!

Ross Kukulinski

kukulinski.com | @rosskukulinski

NodeJS, StatsD, and Graphite

Making technical decisions with metrics