add support for an optional handler for each pin log
great for analyzing which files are most in need of optimizationpull/1/head
parent
a172bad30f
commit
65f88617cf
18
lib/pins.js
18
lib/pins.js
|
@ -28,7 +28,8 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) {
|
|||
// it's a weird API but it's faster than unpinning manually
|
||||
var pins = ref.pins = {};
|
||||
ref.index = 0;
|
||||
ref.latest = 0;
|
||||
ref.latest = 0; // the latest message (timestamp in ms)
|
||||
ref.surplus = 0; // how many lines exist behind a reset
|
||||
return function (line) {
|
||||
ref.index++;
|
||||
if (!Boolean(line)) { return; }
|
||||
|
@ -52,6 +53,7 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) {
|
|||
case 'RESET': {
|
||||
pins = ref.pins = {};
|
||||
if (l[1] && l[1].length) { l[1].forEach((x) => { ref.pins[x] = 1; }); }
|
||||
ref.surplus = ref.index;
|
||||
//jshint -W086
|
||||
// fallthrough
|
||||
}
|
||||
|
@ -90,9 +92,14 @@ Pins.calculateFromLog = function (pinFile, fileName) {
|
|||
pins/A+/A+hyhrQLrgYixOomZYxpuEhwfiVzKk1bBp+arH-zbgo=.ndjson
|
||||
*/
|
||||
|
||||
const getSafeKeyFromPath = function (path) {
|
||||
return path.replace(/^.*\//, '').replace(/\.ndjson/, '');
|
||||
}
|
||||
|
||||
Pins.list = function (done, config) {
|
||||
const pinPath = config.pinPath || './data/pins';
|
||||
const plan = Plan(5);
|
||||
const plan = Plan(config.workers || 5);
|
||||
const handler = config.handler || function () {};
|
||||
|
||||
// TODO externalize this via optional handlers?
|
||||
const stats = {
|
||||
|
@ -106,8 +113,12 @@ Pins.list = function (done, config) {
|
|||
console.log(label, info);
|
||||
};
|
||||
|
||||
const pinned = {};
|
||||
|
||||
// TODO replace this with lib-readline?
|
||||
const streamFile = function (path, cb) {
|
||||
const id = getSafeKeyFromPath(path);
|
||||
|
||||
return void Fs.readFile(path, 'utf8', function (err, body) {
|
||||
if (err) { return void cb(err); }
|
||||
const ref = {};
|
||||
|
@ -115,6 +126,7 @@ Pins.list = function (done, config) {
|
|||
var lines = body.split('\n');
|
||||
stats.lines += lines.length;
|
||||
lines.forEach(pinHandler);
|
||||
handler(ref, id, pinned);
|
||||
cb(void 0, ref);
|
||||
});
|
||||
};
|
||||
|
@ -130,8 +142,6 @@ Pins.list = function (done, config) {
|
|||
});
|
||||
};
|
||||
|
||||
const pinned = {};
|
||||
|
||||
scanDirectory(pinPath, function (err, paths) {
|
||||
if (err) { return; } // XXX
|
||||
paths.forEach(function (path) {
|
||||
|
|
|
@ -3,32 +3,43 @@ const Pins = require("../../lib/pins");
|
|||
|
||||
var stats = {
|
||||
users: 0,
|
||||
lines: 0,
|
||||
pinned: 0,
|
||||
events: 0,
|
||||
lines: 0, // how many lines did you iterate over
|
||||
surplus: 0, // how many of those lines were not needed?
|
||||
pinned: 0, // how many files are pinned?
|
||||
duplicated: 0,
|
||||
};
|
||||
|
||||
var handler = function (ref, id /* safeKey */, pinned) {
|
||||
if (ref.surplus) {
|
||||
//console.log("%s has %s trimmable lines", id, ref.surplus);
|
||||
stats.surplus += ref.surplus;
|
||||
}
|
||||
|
||||
for (var item in ref.pins) {
|
||||
if (!pinned.hasOwnProperty(item)) {
|
||||
//console.log("> %s is pinned", item);
|
||||
stats.pinned++;
|
||||
} else {
|
||||
//console.log("> %s was already pinned", item);
|
||||
stats.duplicated++;
|
||||
}
|
||||
}
|
||||
|
||||
stats.users++;
|
||||
stats.lines += ref.index;
|
||||
//console.log(ref, id);
|
||||
};
|
||||
|
||||
Pins.list(function (err, pinned) {
|
||||
/*
|
||||
for (var id in pinned) {
|
||||
console.log(id);
|
||||
stats.pinned++;
|
||||
}
|
||||
*/
|
||||
console.log(stats);
|
||||
}, {
|
||||
pinPath: require("../../lib/load-config").pinPath
|
||||
pinPath: require("../../lib/load-config").pinPath,
|
||||
handler: handler,
|
||||
});
|
||||
|
||||
/*
|
||||
function (ref, safeKey, pinned) {
|
||||
stats.users++;
|
||||
stats.lines += ref.index;
|
||||
|
||||
Object.keys(ref.pins).forEach(function (id) {
|
||||
if (!pinned[id]) {
|
||||
pinned[id] = true;
|
||||
stats.pinned++;
|
||||
}
|
||||
});
|
||||
//console.log("pin", stats.events++);
|
||||
//console.log(ref, safeKey);
|
||||
}*/
|
||||
|
|
Loading…
Reference in New Issue