add support for an optional handler for each pin log

great for analyzing which files are most in need of optimization

lib/pins.js

@@ -28,7 +28,8 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) {
     // it's a weird API but it's faster than unpinning manually
     var pins = ref.pins = {};
     ref.index = 0;
-    ref.latest = 0;
+    ref.latest = 0; // the latest message (timestamp in ms)
+    ref.surplus = 0; // how many lines exist behind a reset
     return function (line) {
         ref.index++;
         if (!Boolean(line)) { return; }
@@ -52,6 +53,7 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) {
             case 'RESET': {
                 pins = ref.pins = {};
                 if (l[1] && l[1].length) { l[1].forEach((x) => { ref.pins[x] = 1; }); }
+                ref.surplus = ref.index;
                 //jshint -W086
                 // fallthrough
             }
@@ -90,9 +92,14 @@ Pins.calculateFromLog = function (pinFile, fileName) {
     pins/A+/A+hyhrQLrgYixOomZYxpuEhwfiVzKk1bBp+arH-zbgo=.ndjson
 */
 
+const getSafeKeyFromPath = function (path) {
+    return path.replace(/^.*\//, '').replace(/\.ndjson/, '');
+}
+
 Pins.list = function (done, config) {
     const pinPath = config.pinPath || './data/pins';
-    const plan = Plan(5);
+    const plan = Plan(config.workers || 5);
+    const handler = config.handler || function () {};
 
     // TODO externalize this via optional handlers?
     const stats = {
@@ -106,8 +113,12 @@ Pins.list = function (done, config) {
         console.log(label, info);
     };
 
+    const pinned = {};
+
     // TODO replace this with lib-readline?
     const streamFile = function (path, cb) {
+        const id = getSafeKeyFromPath(path);
+
         return void Fs.readFile(path, 'utf8', function (err, body) {
             if (err) { return void cb(err); }
             const ref = {};
@@ -115,6 +126,7 @@ Pins.list = function (done, config) {
             var lines = body.split('\n');
             stats.lines += lines.length;
             lines.forEach(pinHandler);
+            handler(ref, id, pinned);
             cb(void 0, ref);
         });
     };
@@ -130,8 +142,6 @@ Pins.list = function (done, config) {
         });
     };
 
-    const pinned = {};
-
     scanDirectory(pinPath, function (err, paths) {
         if (err) { return; } // XXX
         paths.forEach(function (path) {

scripts/tests/test-pins.js

@@ -3,32 +3,43 @@ const Pins = require("../../lib/pins");
 
 var stats = {
     users: 0,
-    lines: 0,
+    lines: 0, // how many lines did you iterate over
-    pinned: 0,
+    surplus: 0, // how many of those lines were not needed?
-    events: 0,
+    pinned: 0, // how many files are pinned?
+    duplicated: 0,
 };
 
-Pins.list(function (err, pinned) {
+var handler = function (ref, id /* safeKey */, pinned) {
-    for (var id in pinned) {
+    if (ref.surplus) {
-        console.log(id);
+        //console.log("%s has %s trimmable lines", id, ref.surplus);
+        stats.surplus += ref.surplus;
+    }
+
+    for (var item in ref.pins) {
+        if (!pinned.hasOwnProperty(item)) {
+            //console.log("> %s is pinned", item);
             stats.pinned++;
+        } else {
+            //console.log("> %s was already pinned", item);
+            stats.duplicated++;
+        }
     }
-    console.log(stats);
-}, {
-    pinPath: require("../../lib/load-config").pinPath
-});
 
-/*
-function (ref, safeKey, pinned) {
     stats.users++;
     stats.lines += ref.index;
+    //console.log(ref, id);
+};
 
-    Object.keys(ref.pins).forEach(function (id) {
+Pins.list(function (err, pinned) {
-        if (!pinned[id]) {
+/*
-            pinned[id] = true;
+    for (var id in pinned) {
+        console.log(id);
         stats.pinned++;
     }
+*/
+    console.log(stats);
+}, {
+    pinPath: require("../../lib/load-config").pinPath,
+    handler: handler,
 });
-    //console.log("pin", stats.events++);
+
-    //console.log(ref, safeKey);
-}*/