diff --git a/lib/pins.js b/lib/pins.js index 23b1364a3..96743f55a 100644 --- a/lib/pins.js +++ b/lib/pins.js @@ -2,6 +2,11 @@ var Pins = module.exports; +const Fs = require("fs"); +const Path = require("path"); +const Util = require("./common-util"); +const Plan = require("./plan"); + /* Accepts a reference to an object, and... either a string describing which log is being processed (backwards compatibility), or a function which will log the error with all relevant data @@ -22,7 +27,10 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) { // make sure to get ref.pins as the result // it's a weird API but it's faster than unpinning manually var pins = ref.pins = {}; + ref.index = 0; + ref.latest = 0; return function (line) { + ref.index++; if (!Boolean(line)) { return; } var l; @@ -36,6 +44,10 @@ var createLineHandler = Pins.createLineHandler = function (ref, errorHandler) { return void errorHandler('PIN_LINE_NOT_FORMAT_ERROR', l); } + if (typeof(l[2]) === 'number') { + ref.latest = l[2]; // date + } + switch (l[0]) { case 'RESET': { pins = ref.pins = {}; @@ -72,5 +84,87 @@ Pins.calculateFromLog = function (pinFile, fileName) { return Object.keys(ref.pins); }; -// TODO refactor to include a streaming version for use in rpc.js as well +/* + pins/ + pins/A+/ + pins/A+/A+hyhrQLrgYixOomZYxpuEhwfiVzKk1bBp+arH-zbgo=.ndjson +*/ + +Pins.list = function (done, config) { + const pinPath = config.pinPath || './data/pins'; + const plan = Plan(5); + + // TODO externalize this via optional handlers? + const stats = { + logs: 0, + dirs: 0, + pinned: 0, + lines: 0, + }; + + const errorHandler = function (label, info) { + console.log(label, info); + }; + + // TODO replace this with lib-readline? + const streamFile = function (path, cb) { + return void Fs.readFile(path, 'utf8', function (err, body) { + if (err) { return void cb(err); } + const ref = {}; + const pinHandler = createLineHandler(ref, errorHandler); + var lines = body.split('\n'); + stats.lines += lines.length; + lines.forEach(pinHandler); + cb(void 0, ref); + }); + }; + + const scanDirectory = function (path, cb) { + Fs.readdir(path, function (err, list) { + if (err) { + return void cb(err); + } + cb(void 0, list.map(function (item) { + return Path.join(path, item); + })); + }); + }; + + const pinned = {}; + + scanDirectory(pinPath, function (err, paths) { + if (err) { return; } // XXX + paths.forEach(function (path) { + plan.job(1, function (next) { + scanDirectory(path, function (nested_err, nested_paths) { + if (nested_err) { return; } // XXX + stats.dirs++; + nested_paths.forEach(function (nested_path) { + if (!/\.ndjson$/.test(nested_path)) { return; } + plan.job(0, function (next) { + streamFile(nested_path, function (err, ref) { + if (err) { return; } // XXX + stats.logs++; + var set = ref.pins; + for (var item in set) { + if (!pinned.hasOwnProperty(item)) { + pinned[item] = true; + stats.pinned++; + } + } + next(); + }); + }); + }); + next(); + }); + }); + }); + + plan.done(function () { + // err ? + done(void 0, pinned); + }).start(); + }); +};