diff --git a/lib/pins.js b/lib/pins.js index 96743f55a..cb3d3f0c7 100644 --- a/lib/pins.js +++ b/lib/pins.js @@ -28,7 +28,8 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) { // it's a weird API but it's faster than unpinning manually var pins = ref.pins = {}; ref.index = 0; - ref.latest = 0; + ref.latest = 0; // the latest message (timestamp in ms) + ref.surplus = 0; // how many lines exist behind a reset return function (line) { ref.index++; if (!Boolean(line)) { return; } @@ -52,6 +53,7 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) { case 'RESET': { pins = ref.pins = {}; if (l[1] && l[1].length) { l[1].forEach((x) => { ref.pins[x] = 1; }); } + ref.surplus = ref.index; //jshint -W086 // fallthrough } @@ -90,9 +92,14 @@ Pins.calculateFromLog = function (pinFile, fileName) { pins/A+/A+hyhrQLrgYixOomZYxpuEhwfiVzKk1bBp+arH-zbgo=.ndjson */ +const getSafeKeyFromPath = function (path) { + return path.replace(/^.*\//, '').replace(/\.ndjson/, ''); +} + Pins.list = function (done, config) { const pinPath = config.pinPath || './data/pins'; - const plan = Plan(5); + const plan = Plan(config.workers || 5); + const handler = config.handler || function () {}; // TODO externalize this via optional handlers? const stats = { @@ -106,8 +113,12 @@ Pins.list = function (done, config) { console.log(label, info); }; + const pinned = {}; + // TODO replace this with lib-readline? const streamFile = function (path, cb) { + const id = getSafeKeyFromPath(path); + return void Fs.readFile(path, 'utf8', function (err, body) { if (err) { return void cb(err); } const ref = {}; @@ -115,6 +126,7 @@ Pins.list = function (done, config) { var lines = body.split('\n'); stats.lines += lines.length; lines.forEach(pinHandler); + handler(ref, id, pinned); cb(void 0, ref); }); }; @@ -130,8 +142,6 @@ Pins.list = function (done, config) { }); }; - const pinned = {}; - scanDirectory(pinPath, function (err, paths) { if (err) { return; } // XXX paths.forEach(function (path) { diff --git a/scripts/tests/test-pins.js b/scripts/tests/test-pins.js index 8697d238b..eea164230 100644 --- a/scripts/tests/test-pins.js +++ b/scripts/tests/test-pins.js @@ -3,32 +3,43 @@ const Pins = require("../../lib/pins"); var stats = { users: 0, - lines: 0, - pinned: 0, - events: 0, + lines: 0, // how many lines did you iterate over + surplus: 0, // how many of those lines were not needed? + pinned: 0, // how many files are pinned? + duplicated: 0, +}; + +var handler = function (ref, id /* safeKey */, pinned) { + if (ref.surplus) { + //console.log("%s has %s trimmable lines", id, ref.surplus); + stats.surplus += ref.surplus; + } + + for (var item in ref.pins) { + if (!pinned.hasOwnProperty(item)) { + //console.log("> %s is pinned", item); + stats.pinned++; + } else { + //console.log("> %s was already pinned", item); + stats.duplicated++; + } + } + + stats.users++; + stats.lines += ref.index; + //console.log(ref, id); }; Pins.list(function (err, pinned) { +/* for (var id in pinned) { console.log(id); stats.pinned++; } +*/ console.log(stats); }, { - pinPath: require("../../lib/load-config").pinPath + pinPath: require("../../lib/load-config").pinPath, + handler: handler, }); -/* -function (ref, safeKey, pinned) { - stats.users++; - stats.lines += ref.index; - - Object.keys(ref.pins).forEach(function (id) { - if (!pinned[id]) { - pinned[id] = true; - stats.pinned++; - } - }); - //console.log("pin", stats.events++); - //console.log(ref, safeKey); -}*/