add support for an optional handler for each pin log

great for analyzing which files are most in need of optimization
pull/1/head
ansuz 5 years ago
parent a172bad30f
commit 65f88617cf

@ -28,7 +28,8 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) {
// it's a weird API but it's faster than unpinning manually
var pins = ref.pins = {};
ref.index = 0;
ref.latest = 0;
ref.latest = 0; // the latest message (timestamp in ms)
ref.surplus = 0; // how many lines exist behind a reset
return function (line) {
ref.index++;
if (!Boolean(line)) { return; }
@ -52,6 +53,7 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) {
case 'RESET': {
pins = ref.pins = {};
if (l[1] && l[1].length) { l[1].forEach((x) => { ref.pins[x] = 1; }); }
ref.surplus = ref.index;
//jshint -W086
// fallthrough
}
@ -90,9 +92,14 @@ Pins.calculateFromLog = function (pinFile, fileName) {
pins/A+/A+hyhrQLrgYixOomZYxpuEhwfiVzKk1bBp+arH-zbgo=.ndjson
*/
const getSafeKeyFromPath = function (path) {
return path.replace(/^.*\//, '').replace(/\.ndjson/, '');
}
Pins.list = function (done, config) {
const pinPath = config.pinPath || './data/pins';
const plan = Plan(5);
const plan = Plan(config.workers || 5);
const handler = config.handler || function () {};
// TODO externalize this via optional handlers?
const stats = {
@ -106,8 +113,12 @@ Pins.list = function (done, config) {
console.log(label, info);
};
const pinned = {};
// TODO replace this with lib-readline?
const streamFile = function (path, cb) {
const id = getSafeKeyFromPath(path);
return void Fs.readFile(path, 'utf8', function (err, body) {
if (err) { return void cb(err); }
const ref = {};
@ -115,6 +126,7 @@ Pins.list = function (done, config) {
var lines = body.split('\n');
stats.lines += lines.length;
lines.forEach(pinHandler);
handler(ref, id, pinned);
cb(void 0, ref);
});
};
@ -130,8 +142,6 @@ Pins.list = function (done, config) {
});
};
const pinned = {};
scanDirectory(pinPath, function (err, paths) {
if (err) { return; } // XXX
paths.forEach(function (path) {

@ -3,32 +3,43 @@ const Pins = require("../../lib/pins");
var stats = {
users: 0,
lines: 0,
pinned: 0,
events: 0,
lines: 0, // how many lines did you iterate over
surplus: 0, // how many of those lines were not needed?
pinned: 0, // how many files are pinned?
duplicated: 0,
};
Pins.list(function (err, pinned) {
for (var id in pinned) {
console.log(id);
var handler = function (ref, id /* safeKey */, pinned) {
if (ref.surplus) {
//console.log("%s has %s trimmable lines", id, ref.surplus);
stats.surplus += ref.surplus;
}
for (var item in ref.pins) {
if (!pinned.hasOwnProperty(item)) {
//console.log("> %s is pinned", item);
stats.pinned++;
} else {
//console.log("> %s was already pinned", item);
stats.duplicated++;
}
}
console.log(stats);
}, {
pinPath: require("../../lib/load-config").pinPath
});
/*
function (ref, safeKey, pinned) {
stats.users++;
stats.lines += ref.index;
//console.log(ref, id);
};
Object.keys(ref.pins).forEach(function (id) {
if (!pinned[id]) {
pinned[id] = true;
Pins.list(function (err, pinned) {
/*
for (var id in pinned) {
console.log(id);
stats.pinned++;
}
*/
console.log(stats);
}, {
pinPath: require("../../lib/load-config").pinPath,
handler: handler,
});
//console.log("pin", stats.events++);
//console.log(ref, safeKey);
}*/

Loading…
Cancel
Save