NodeJS, StatsD, and Graphite

Making technical decisions with metrics

Created by Ross Kukulinski / @rosskukulinski

Engineers

Optimize all the things

but...

We're building complex systems

so...

How do you know what to optimize?

How do you know if you've been successful?

Metrics

allow you to explore

allow you to determine what's important

allow you to focus on what's important

enable educated decisions

Example from production

Graphite

  • Stores numeric time-series data
  • Renders graphs of this data on demand
  • Exports data via REST API

Graphite Architecture

  • carbon - daemon that listens for time-series data
  • whisper - database for storing data
  • webapp - webapp that renders graphs
Graphite web app

Data retention

  • How often can you produce your data?
  • What is the finest precision you will require?
  • How long will you need that level of precision?
  • What is the coarsest precision you can use?
  • How far back would you ever need to see data?

StatsD

  • NodeJS UDP network daemon
  • Listens for stats and aggregates before sending to graphite
  • Provides support for different types of metrics

NodeJS StatsD Clients

Getting started with Lynx

Install

npm install lynx

example1.js:

var lynx = require('lynx');
var opt = {}; // opt.prefix = 'myPrefix';
var metrics = new lynx('162.242.212.227', 8125, opt); // StatsD IP & Port
metrics.gauge('example1.gauge', 300); // Send our first metric

Metric Types

  • Gauge
  • Counter
  • Timer
  • Set

Gauges


// example2.js
function updateGauges() {
  metrics.gauge('example2.gauge1', Math.random()+3);
  metrics.gauge('example2.gauge2', Math.random());
  setTimeout(updateGauges, 500);
}
updateGauges();

Counters


//example3.js
function increment() {
  for (var j=10; j‹Math.random()*40; j++) {
    metrics.increment('example3.counter2');
    // metrics.decrement('example3.counter2');
  }
  setTimeout(increment, 500);
}
increment();

Timers


//example4.js
function timer() {
  for (var i=0; i‹20; i++) {
    metrics.timing('example4.timer1', Math.random()*50); // time in ms
  }
  setTimeout(timer, 500);
}
timer();

//example5.js - Timers Redux
function timer() {
  var timer = metrics.createTimer('example5.timer1');
  setTimeout(function(){timer.stop()}, Math.random()*25);
}
function timers() {
  for (var i=0; i‹20; i++) {
    timer();
  }
  setTimeout(timers, 500);
}
timers();

Sets


// example6.js
function pick() {
  metrics.set('example6.user', 'ross');
  metrics.set('example6.user', 'ross');
  if (Math.random() > 0.9) { metrics.set('example6.user', 'jared'); }
  setTimeout(pick, 1000);
}
pick();

Sampling

'really, really frequent' data can overwhelm StatsD


  metrics.increment('some.counter', 0.1); // Send only 10% of calls
          

lynx-express

Install
npm install lynx-express
Use Middleware

// Import lynx and lynx-express
var lynx = require('lynx');
var lynxExpress = require('lynx-express');

// Setup your lynx StatsD client as normal
var metrics = new lynx('162.242.212.227', 8125, {prefix: 'express'});

// Create the lynx Express middleware
var lynxMiddleware = lynxExpress(metrics);

// Tell Express to use your lynx middleware
server.use(lynxMiddleware());
// or server.use(lynxMiddleware({timeByUrl: true});
          

Express Server

// Express server
var express = require('express');
var app = express();
setupLynxMiddleware(app);
function randResp(req, res, next) {
  var delay = 1000;
  if (Math.random() > 0.95) {
    delay = delay*2;
  }
  setTimeout(function() {res.send(req.url);}, Math.random()*delay);
};
app.get('/', randResp);
app.get('/users/', randResp);
app.get('/users/:user', randResp);
app.listen(3000);

Timing For Each Route

Express Route Timing

Graphite demo

Configuration changes


# Changes to /opt/graphite/webapp/content/js/composer_widgets.js
DEFAULT_CACHE_DURATION = 1 # Cache images and data for 1 minute  
          

// Changes to /opt/graphite/webapp/content/js/composer_widgets.js
var doRefresh = function () {
  Composer.updateImage();
  var interval = 1; // Change refresh rate for widgets to every second                                                                                  
  button.timer = setTimeout(doRefresh, interval * 1000)
}  
          

// Changes to /etc/statsd/config.js
"flushInterval": 1000, 
          

Thanks!

Ross Kukulinski

kukulinski.com | @rosskukulinski