Open-Sourcing indy!

Author: tomgco

.gitignore

@@ -0,0 +1,7 @@
+# Vim
+*.swp
+*.swo
+
+# Node
+node_modules
+

.tern-project

@@ -0,0 +1,10 @@
+{
+  "libs": [
+  ],
+  "loadEagerly": [
+  ],
+  "plugins": {
+    "node": {
+    }
+  }
+}

README.md

@@ -0,0 +1,90 @@
+# Indy
+
+Fancy having your own Dr. Henry Walton "Indiana" Jones, Jr? Now you can!
+
+
+```javascript
+
+var Indy = require('indy');
+
+var indy = new Indy();
+
+// Defaults to our artefact custom event dispatcher.
+// Times eventloop delay, process info, memory info, etc.
+indy.whip( { /* dispatcher: new StatsD() */  } );
+
+// Not sure why you would need it yet but, lets just see.
+indy.pause();
+indy.resume();
+
+// Dumps a metric shit-ton of things, memory, core, yada yada ya.
+// Breaks things, blocks the eventloop etc. not fun.
+indy.extract(); // heart!
+
+// We always forget this!
+indy.hat();
+
+```
+
+## Example output from the default dispatcher
+
+```
+{
+  "gccount": 0, // delta of gc counts between each capture
+  "memrss": 18083840, // bytes
+  "memfree": 721039360, // bytes
+  "memtotal": 4076998656, // bytes
+  "memheaptotal": 14519296, // bytes
+  "memheapused": 5660016, // bytes
+  "cpus": [
+    {
+      "model": "intel(r) core(tm) i5-3427u cpu @ 1.80ghz",
+      "speed": 800, // Mhz
+      "times": {
+        "user": 11770900, // ms
+        "nice": 1509700, // ms
+        "sys": 4852500, // ms
+        "idle": 44234800, // ms
+        "irq": 700
+      }
+    },
+    {
+      "model": "intel(r) core(tm) i5-3427u cpu @ 1.80ghz",
+      "speed": 800,
+      "times": {
+        "user": 9865300,
+        "nice": 1292400,
+        "sys": 4128400,
+        "idle": 48376000,
+        "irq": 0
+      }
+    },
+    {
+      "model": "intel(r) core(tm) i5-3427u cpu @ 1.80ghz",
+      "speed": 800,
+      "times": {
+        "user": 11763300,
+        "nice": 1567500,
+        "sys": 5049500,
+        "idle": 42676900,
+        "irq": 0
+      }
+    },
+    {
+      "model": "intel(r) core(tm) i5-3427u cpu @ 1.80ghz",
+      "speed": 1000,
+      "times": {
+        "user": 9685200,
+        "nice": 1140400,
+        "sys": 4383000,
+        "idle": 45695100,
+        "irq": 0
+      }
+    }
+  ],
+  "loadavg5": 0.6865234375,
+  "loadavg10": 1.10400390625,
+  "loadavg15": 1.39208984375,
+  "delay": 3.174327 //ms
+}
+```

index.js

@@ -0,0 +1,70 @@
+var EventEmitter = require('events').EventEmitter;
+var util = require('util');
+var DefaultDispatcher = require('./lib/dispatcher');
+var async = require('async');
+
+var delay = require('./lib/loop-delay');
+var memory = require('./lib/memory');
+var GC = require('./lib/gc');
+var cpu = require('./lib/cpu');
+
+function Indy(options) {
+  EventEmitter.call(this);
+
+  this.options = options || {};
+  this.dispatcher = this.options.dispatcher || new DefaultDispatcher();
+  this.interval = this.options.interval || 10000; // in ms
+  this.intervalRef = undefined;
+  this.gc = new GC();
+}
+
+util.inherits(Indy, EventEmitter);
+
+// Start watching for any events and publish them to our dispatcher.
+
+Indy.prototype.whip = function () {
+  this.gc.start();
+  var self = this;
+  var collect = function () {
+    var metrics =
+      { delay: delay,
+        gcCount: self.gc.delta.bind(self.gc),
+        memRss: memory.rss,
+        memFree: memory.free,
+        memTotal: memory.total,
+        memHeapTotal: memory.heapTotal,
+        memHeapUsed: memory.heapUsed,
+        cpus: cpu.cpus,
+        loadavg5: cpu.loadavg5,
+        loadavg10: cpu.loadavg10,
+        loadavg15: cpu.loadavg15
+      };
+
+    async.parallelLimit(metrics, 2, function (err, data) {
+      if (err) {
+        return;
+      }
+      self.dispatcher.dispatch(data);
+    });
+  };
+
+  self.intervalRef = setInterval(collect, self.interval);
+};
+
+Indy.prototype.pause = function (done) {
+  var self = this;
+  if (!self.intervalRef) {
+    return;
+  }
+  clearInterval(self.intervalRef);
+  done();
+};
+
+Indy.prototype.resume = function () {
+  if (this.intervalRef) {
+    return;
+  }
+  this.whip();
+};
+
+module.exports = Indy;

lib/bench.js

@@ -0,0 +1,22 @@
+function Bench() {
+  if (!(this instanceof Bench)) {
+    return new Bench();
+  }
+  this.ts = 0;
+
+  this.start();
+}
+
+Bench.prototype = {};
+
+Bench.prototype.start = function () {
+  this.ts = process.hrtime();
+};
+
+Bench.prototype.elapsed = function () {
+  var ts = process.hrtime(this.ts);
+  //     Secs into ms  + ns into ms
+  return (ts[0] * 1e3) + (ts[1] / 1e6);
+};
+
+module.exports = Bench;

lib/cpu.js

@@ -0,0 +1,9 @@
+var os = require('os');
+var wrap = require('./util').wrap;
+
+module.exports =
+  { cpus: wrap(os.cpus()),
+    loadavg5: wrap(os.loadavg()[0]),
+    loadavg10: wrap(os.loadavg()[1]),
+    loadavg15: wrap(os.loadavg()[2])
+  };

lib/dispatcher.js

@@ -0,0 +1,19 @@
+var EventEmitter = require('events').EventEmitter;
+var util = require('util');
+
+function DefaultDispatcher(options) {
+  if (!(this instanceof DefaultDispatcher)) {
+    return new DefaultDispatcher(options);
+  }
+  EventEmitter.call(this);
+
+  this.options = options || {};
+}
+
+util.inherits(DefaultDispatcher, EventEmitter);
+
+DefaultDispatcher.prototype.dispatch = function (obj) {
+  process.stdout.write(JSON.stringify(obj, true, 2));
+};
+
+module.exports = DefaultDispatcher;

lib/gc.js

@@ -0,0 +1,67 @@
+var EventEmitter = require('events').EventEmitter;
+var v8 = new EventEmitter();
+
+try {
+  v8 = require('tracing').v8;
+  if (!process.env.NODE_ENV) {
+    v8.setMaxListeners(0);
+  }
+} catch (e) {
+  console.log('Failed to load v8, not collecting GC information');
+}
+
+// TODO: Need to decide on if we should do some internal buffering of the
+// object, or just send the last gc event since the last probe.
+// Most likely the latter as under high loads we don't want artefact or indy
+// to exacerbate an issue.
+function V8Gc() {
+  if (!(this instanceof V8Gc)) {
+    return new V8Gc();
+  }
+  this.count = 0;
+  this.started = false;
+  this.start();
+}
+
+V8Gc.prototype.start = function () {
+  if(this.started) {
+    return false;
+  }
+  this.started = true;
+
+  var self = this;
+  // need to ensure we don't over use this.
+  // possible major performance bottle-neck.
+  v8.on('gc', function onGC(before, after) {
+    // we need to be super defensive here against changing v8 versions.
+    // and breaking api changes to that subsystem.
+
+    self.count++;
+    // undefined behaviour on the first gc.
+    // bail out early.
+    if (!before || !after) {
+      return false;
+    }
+
+    // increment gc counter.
+    // notify what gc occured.
+    // send gc sizes.
+
+    // This is very handy to check that a large gc has occured
+    // and has caused a significant load on the CPU.
+
+  });
+
+};
+
+V8Gc.prototype.reset = function () {
+  this.count = 0;
+};
+
+V8Gc.prototype.delta = function (cb) {
+  var delta = this.count;
+  this.count = 0;
+  return cb(null, delta);
+};
+
+module.exports = V8Gc;

lib/loop-delay.js

@@ -0,0 +1,8 @@
+var Bench = require('./bench');
+
+module.exports = function loopDelay(cb) {
+  var bench = new Bench();
+  setImmediate(function measure() {
+    return cb(null, bench.elapsed());
+  });
+};

lib/memory.js

@@ -0,0 +1,12 @@
+var wrap = require('./util').wrap;
+var os = require('os');
+
+// System
+exports.rss = wrap(process.memoryUsage().rss);
+exports.total = wrap(os.totalmem());
+exports.free = wrap(os.freemem());
+
+// V8 Memory
+exports.heapTotal = wrap(process.memoryUsage().heapTotal);
+exports.heapUsed = wrap(process.memoryUsage().heapUsed);
+

lib/util.js

@@ -0,0 +1,5 @@
+exports.wrap = function wrap(result) {
+  return function wrappedFn(cb) {
+    return cb(null, result);
+  };
+};

package.json

@@ -0,0 +1,17 @@
+{
+  "name": "indy",
+  "version": "1.0.0",
+  "description": "discover all the artifacts!",
+  "main": "lib/indy.js",
+  "scripts": {
+    "test": "./node_modules/.bin/mocha test"
+  },
+  "author": "Tom Gallacher <[email protected]>",
+  "license": "Artistic-2.0",
+  "dependencies": {
+    "async": "^0.9.0"
+  },
+  "devDependencies": {
+    "mocha": "^1.20.1"
+  }
+}