From 5c9cd5ffc679911deb517ab863de715aabdf4f93 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 17 Apr 2019 14:46:46 +0200 Subject: [PATCH] standardize pin calculation routine --- lib/pins.js | 43 +++++++++++++++++++++++++++++++ scripts/check-account-deletion.js | 27 ++----------------- scripts/pinned.js | 35 ++----------------------- scripts/pinneddata.js | 33 ++---------------------- 4 files changed, 49 insertions(+), 89 deletions(-) create mode 100644 lib/pins.js diff --git a/lib/pins.js b/lib/pins.js new file mode 100644 index 000000000..4e0791d05 --- /dev/null +++ b/lib/pins.js @@ -0,0 +1,43 @@ +/*jshint esversion: 6 */ + +var Pins = module.exports; + +/* + takes contents of a pinFile (UTF8 string) + and the pin file's name + returns an array of of channel ids which are pinned + + throw errors on pin logs with invalid pin data +*/ +Pins.calculateFromLog = function (pinFile, fileName) { + var pins = {}; + pinFile.split('\n').filter((x)=>(x)).map((l) => JSON.parse(l)).forEach((l) => { + switch (l[0]) { + case 'RESET': { + pins = {}; + if (l[1] && l[1].length) { l[1].forEach((x) => { pins[x] = 1; }); } + //jshint -W086 + // fallthrough + } + case 'PIN': { + l[1].forEach((x) => { pins[x] = 1; }); + break; + } + case 'UNPIN': { + l[1].forEach((x) => { delete pins[x]; }); + break; + } + default: + // FIXME logging + // TODO write to the error log + /* Log.error('CORRUPTED_PIN_LOG', { + line: JSON.stringify(l), + fileName: fileName, + }); */ + console.error(new Error (JSON.stringify(l) + ' ' + fileName)); + } + }); + return Object.keys(pins); +}; + +// TODO refactor to include a streaming version for use in rpc.js as well diff --git a/scripts/check-account-deletion.js b/scripts/check-account-deletion.js index 69d720cb5..020e3c254 100644 --- a/scripts/check-account-deletion.js +++ b/scripts/check-account-deletion.js @@ -4,32 +4,9 @@ const nThen = require('nthen'); const Pinned = require('./pinned'); const Nacl = require('tweetnacl'); const Path = require('path'); +const Pins = require('../lib/pins'); const Config = require('../lib/load-config'); -const hashesFromPinFile = (pinFile, fileName) => { - var pins = {}; - pinFile.split('\n').filter((x)=>(x)).map((l) => JSON.parse(l)).forEach((l) => { - switch (l[0]) { - case 'RESET': { - pins = {}; - if (l[1] && l[1].length) { l[1].forEach((x) => { pins[x] = 1; }); } - //jshint -W086 - // fallthrough - } - case 'PIN': { - l[1].forEach((x) => { pins[x] = 1; }); - break; - } - case 'UNPIN': { - l[1].forEach((x) => { delete pins[x]; }); - break; - } - default: throw new Error(JSON.stringify(l) + ' ' + fileName); - } - }); - return Object.keys(pins); -}; - var escapeKeyCharacters = function (key) { return key && key.replace && key.replace(/\//g, '-'); }; @@ -61,7 +38,7 @@ nThen((waitFor) => { let f = Path.join(pinPath, edPublic.slice(0, 2), edPublic + '.ndjson'); Fs.readFile(f, waitFor((err, content) => { if (err) { throw err; } - pinned = hashesFromPinFile(content.toString('utf8'), f); + pinned = Pins.calculateFromLog(content.toString('utf8'), f); })); }).nThen((waitFor) => { Pinned.load(waitFor((d) => { diff --git a/scripts/pinned.js b/scripts/pinned.js index ba1054d77..f42b4bf8c 100644 --- a/scripts/pinned.js +++ b/scripts/pinned.js @@ -4,6 +4,7 @@ const Path = require("path"); const Semaphore = require('saferphore'); const Once = require("../lib/once"); const nThen = require('nthen'); +const Pins = require("../lib/pins"); const sema = Semaphore.create(20); @@ -11,38 +12,6 @@ let dirList; const fileList = []; const pinned = {}; -// FIXME this seems to be duplicated in a few places. -// make it a library and put it in ./lib/ -const checkPinStatus = (pinFile, fileName) => { - var pins = {}; - pinFile.split('\n').filter((x)=>(x)).map((l) => JSON.parse(l)).forEach((l) => { - switch (l[0]) { - case 'RESET': { - pins = {}; - if (l[1] && l[1].length) { l[1].forEach((x) => { pins[x] = 1; }); } - //jshint -W086 - // fallthrough - } - case 'PIN': { - l[1].forEach((x) => { pins[x] = 1; }); - break; - } - case 'UNPIN': { - l[1].forEach((x) => { delete pins[x]; }); - break; - } - default: - // TODO write to the error log - /* Log.error('CORRUPTED_PIN_LOG', { - line: JSON.stringify(l), - fileName: fileName, - }); */ - console.error(new Error (JSON.stringify(l) + ' ' + fileName)); - } - }); - return Object.keys(pins); -}; - module.exports.load = function (cb, config) { var pinPath = config.pinPath || './pins'; var done = Once(cb); @@ -84,7 +53,7 @@ module.exports.load = function (cb, config) { waitFor.abort(); return void done(err); } - const hashes = checkPinStatus(content.toString('utf8'), f); + const hashes = Pins.calculateFromLog(content.toString('utf8'), f); hashes.forEach((x) => { (pinned[x] = pinned[x] || {})[f.replace(/.*\/([^/]*).ndjson$/, (x, y)=>y)] = 1; }); diff --git a/scripts/pinneddata.js b/scripts/pinneddata.js index 5f3d4e1ec..e87aaf971 100644 --- a/scripts/pinneddata.js +++ b/scripts/pinneddata.js @@ -3,36 +3,7 @@ const Fs = require('fs'); const Semaphore = require('saferphore'); const nThen = require('nthen'); const Path = require('path'); - -/* - takes contents of a pinFile (UTF8 string) - and the pin file's name - returns an array of of channel ids which are pinned - - throw errors on pin logs with invalid pin data -*/ -const hashesFromPinFile = (pinFile, fileName) => { - var pins = {}; - pinFile.split('\n').filter((x)=>(x)).map((l) => JSON.parse(l)).forEach((l) => { - switch (l[0]) { - case 'RESET': { - pins = {}; - if (l[1] && l[1].length) { l[1].forEach((x) => { pins[x] = 1; }); } - break; - } - case 'PIN': { - l[1].forEach((x) => { pins[x] = 1; }); - break; - } - case 'UNPIN': { - l[1].forEach((x) => { delete pins[x]; }); - break; - } - default: throw new Error(JSON.stringify(l) + ' ' + fileName); - } - }); - return Object.keys(pins); -}; +const Pins = require('../lib/pins'); /* takes an array of pinned file names @@ -152,7 +123,7 @@ module.exports.load = function (config, cb) { Fs.readFile(f, waitFor(returnAfter((err, content) => { if (err) { throw err; } // get the list of channels pinned by this log - const hashes = hashesFromPinFile(content.toString('utf8'), f); + const hashes = Pins.calculateFromLog(content.toString('utf8'), f); if (config.unpinned) { hashes.forEach((x) => { pinned[x] = 1; }); } else {