diff --git a/lib/pins.js b/lib/pins.js index cb3d3f0c7..3c1bf3967 100644 --- a/lib/pins.js +++ b/lib/pins.js @@ -96,11 +96,20 @@ const getSafeKeyFromPath = function (path) { return path.replace(/^.*\//, '').replace(/\.ndjson/, ''); } -Pins.list = function (done, config) { +Pins.list = function (_done, config) { const pinPath = config.pinPath || './data/pins'; const plan = Plan(config.workers || 5); const handler = config.handler || function () {}; + var isDone = false; + // ensure that 'done' is only called once + // that it calls back asynchronously + // and that it sets 'isDone' to true, so that pending processes + // know to abort + const done = Util.once(Util.both(Util.mkAsync(_done), function () { + isDone = true; + })); + // TODO externalize this via optional handlers? const stats = { logs: 0, @@ -137,29 +146,39 @@ Pins.list = function (done, config) { return void cb(err); } cb(void 0, list.map(function (item) { - return Path.join(path, item); + return { + path: Path.join(path, item), + id: item.replace(/\.ndjson$/, ''), + }; })); }); }; - scanDirectory(pinPath, function (err, paths) { - if (err) { return; } // XXX - paths.forEach(function (path) { + scanDirectory(pinPath, function (err, dirs) { + if (err) { + if (err.code === 'ENOENT') { return void cb(void 0, {}); } + return void done(err); + } + dirs.forEach(function (dir) { plan.job(1, function (next) { - scanDirectory(path, function (nested_err, nested_paths) { - if (nested_err) { return; } // XXX + if (isDone) { return void next(); } + scanDirectory(dir.path, function (nested_err, logs) { + if (nested_err) { + return void done(err); + } stats.dirs++; - nested_paths.forEach(function (nested_path) { - if (!/\.ndjson$/.test(nested_path)) { return; } + logs.forEach(function (log) { + if (!/\.ndjson$/.test(log.path)) { return; } plan.job(0, function (next) { - streamFile(nested_path, function (err, ref) { - if (err) { return; } // XXX + if (isDone) { return void next(); } + streamFile(log.path, function (err, ref) { + if (err) { return void done(err); } stats.logs++; var set = ref.pins; for (var item in set) { + (pinned[item] = pinned[item] || {})[log.id] = 1; if (!pinned.hasOwnProperty(item)) { - pinned[item] = true; stats.pinned++; } }