From 10eed5c46dc6dc63a9348be73e8f3ce112074cb8 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 13 Jan 2020 10:28:28 -0500 Subject: [PATCH 01/53] drop unmaintained flow annotations --- storage/file.js | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/storage/file.js b/storage/file.js index bb65cff43..74f794cca 100644 --- a/storage/file.js +++ b/storage/file.js @@ -867,36 +867,7 @@ var getMessages = function (env, chanName, handler, cb) { }); }; -/*:: -export type ChainPadServer_MessageObj_t = { buff: Buffer, offset: number }; -export type ChainPadServer_Storage_t = { - readMessagesBin: ( - channelName:string, - start:number, - asyncMsgHandler:(msg:ChainPadServer_MessageObj_t, moreCb:()=>void, abortCb:()=>void)=>void, - cb:(err:?Error)=>void - )=>void, - message: (channelName:string, content:string, cb:(err:?Error)=>void)=>void, - messageBin: (channelName:string, content:Buffer, cb:(err:?Error)=>void)=>void, - getMessages: (channelName:string, msgHandler:(msg:string)=>void, cb:(err:?Error)=>void)=>void, - removeChannel: (channelName:string, cb:(err:?Error)=>void)=>void, - closeChannel: (channelName:string, cb:(err:?Error)=>void)=>void, - flushUnusedChannels: (cb:()=>void)=>void, - getChannelSize: (channelName:string, cb:(err:?Error, size:?number)=>void)=>void, - getChannelMetadata: (channelName:string, cb:(err:?Error|string, data:?any)=>void)=>void, - clearChannel: (channelName:string, (err:?Error)=>void)=>void -}; -export type ChainPadServer_Config_t = { - verbose?: boolean, - filePath?: string, - channelExpirationMs?: number, - openFileLimit?: number -}; -*/ -module.exports.create = function ( - conf /*:ChainPadServer_Config_t*/, - cb /*:(store:ChainPadServer_Storage_t)=>void*/ -) { +module.exports.create = function (conf, cb) { var env = { root: conf.filePath || './datastore', archiveRoot: conf.archivePath || './data/archive', From c89595c7bbab8e37c31e3ba420a64d948b0b9664 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 13 Jan 2020 10:32:06 -0500 Subject: [PATCH 02/53] handle all the simple cases where operations on channels should be queued --- storage/file.js | 68 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/storage/file.js b/storage/file.js index 74f794cca..2a8802c8c 100644 --- a/storage/file.js +++ b/storage/file.js @@ -7,6 +7,8 @@ var Path = require("path"); var nThen = require("nthen"); var Semaphore = require("saferphore"); var Util = require("../lib/common-util"); + +const WriteQueue = require("../lib/write-queue"); const Readline = require("readline"); const ToPull = require('stream-to-pull-stream'); const Pull = require('pull-stream'); @@ -169,6 +171,7 @@ var readMessages = function (path, msgHandler, cb) { could have been amended */ var getChannelMetadata = function (Env, channelId, cb) { + // XXX queue var path = mkPath(Env, channelId); // gets metadata embedded in a file @@ -177,6 +180,7 @@ var getChannelMetadata = function (Env, channelId, cb) { // low level method for getting just the dedicated metadata channel var getDedicatedMetadata = function (env, channelId, handler, cb) { + // XXX queue var metadataPath = mkMetadataPath(env, channelId); readMessages(metadataPath, function (line) { if (!line) { return; } @@ -219,6 +223,7 @@ How to proceed */ + // XXX queue nThen(function (w) { // returns the first line of a channel, parsed... getChannelMetadata(env, channelId, w(function (err, data) { @@ -263,6 +268,7 @@ var writeMetadata = function (env, channelId, data, cb) { // TODO see if we could improve performance by using libnewline const NEWLINE_CHR = ('\n').charCodeAt(0); const mkBufferSplit = () => { + // XXX queue lock let remainder = null; return Pull((read) => { return (abort, cb) => { @@ -314,6 +320,7 @@ const mkOffsetCounter = () => { const readMessagesBin = (env, id, start, msgHandler, cb) => { const stream = Fs.createReadStream(mkPath(env, id), { start: start }); let keepReading = true; + // XXX queue lock Pull( ToPull.read(stream), mkBufferSplit(), @@ -837,6 +844,7 @@ var message = function (env, chanName, msg, cb) { // stream messages from a channel log var getMessages = function (env, chanName, handler, cb) { + // XXX queue lock getChannel(env, chanName, function (err, chan) { if (!chan) { cb(err); @@ -878,6 +886,7 @@ module.exports.create = function (conf, cb) { openFiles: 0, openFileLimit: conf.openFileLimit || 2048, }; + var queue = env.queue = WriteQueue(); var it; nThen(function (w) { @@ -899,10 +908,13 @@ module.exports.create = function (conf, cb) { // write a new message to a log message: function (channelName, content, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - message(env, channelName, content, cb); + queue(channelName, function (next) { + message(env, channelName, content, Util.both(cb, next)); + }); }, // iterate over all the messages in a log getMessages: function (channelName, msgHandler, cb) { + // XXX queue lock if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } getMessages(env, channelName, msgHandler, cb); }, @@ -911,10 +923,13 @@ module.exports.create = function (conf, cb) { // write a new message to a log messageBin: (channelName, content, cb) => { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - messageBin(env, channelName, content, cb); + queue(channelName, function (next) { + messageBin(env, channelName, content, Util.both(cb, next)); + }); }, // iterate over the messages in a log readMessagesBin: (channelName, start, asyncMsgHandler, cb) => { + // XXX queue lock if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } readMessagesBin(env, channelName, start, asyncMsgHandler, cb); }, @@ -923,19 +938,23 @@ module.exports.create = function (conf, cb) { // remove a channel and its associated metadata log if present removeChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - removeChannel(env, channelName, function (err) { - cb(err); + queue(channelName, function (next) { + removeChannel(env, channelName, Util.both(cb, next)); }); }, // remove a channel and its associated metadata log from the archive directory removeArchivedChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - removeArchivedChannel(env, channelName, cb); + queue(channelName, function (next) { + removeArchivedChannel(env, channelName, Util.both(cb, next)); + }); }, // clear all data for a channel but preserve its metadata clearChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - clearChannel(env, channelName, cb); + queue(channelName, function (next) { + clearChannel(env, channelName, Util.both(cb, next)); + }); }, // check if a channel exists in the database @@ -943,47 +962,62 @@ module.exports.create = function (conf, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } // construct the path var filepath = mkPath(env, channelName); - channelExists(filepath, cb); + queue(channelName, function (next) { + channelExists(filepath, Util.both(cb, next)); + }); }, // check if a channel exists in the archive isChannelArchived: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } // construct the path var filepath = mkArchivePath(env, channelName); - channelExists(filepath, cb); + queue(channelName, function (next) { + channelExists(filepath, Util.both(cb, next)); + }); }, // move a channel from the database to the archive, along with its metadata archiveChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - archiveChannel(env, channelName, cb); + queue(channelName, function (next) { + archiveChannel(env, channelName, Util.both(cb, next)); + }); }, // restore a channel from the archive to the database, along with its metadata restoreArchivedChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - unarchiveChannel(env, channelName, cb); + queue(channelName, function (next) { + unarchiveChannel(env, channelName, Util.both(cb, next)); + }); }, // METADATA METHODS // fetch the metadata for a channel getChannelMetadata: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - getChannelMetadata(env, channelName, cb); + queue(channelName, function (next) { + getChannelMetadata(env, channelName, Util.both(cb, next)); + }); }, // iterate over lines of metadata changes from a dedicated log readDedicatedMetadata: function (channelName, handler, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - getDedicatedMetadata(env, channelName, handler, cb); + queue(channelName, function (next) { + getDedicatedMetadata(env, channelName, handler, Util.both(cb, next)); + }); }, // iterate over multiple lines of metadata changes readChannelMetadata: function (channelName, handler, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } + // XXX queue readMetadata(env, channelName, handler, cb); }, // write a new line to a metadata log writeMetadata: function (channelName, data, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - writeMetadata(env, channelName, data, cb); + queue(channelName, function (next) { + writeMetadata(env, channelName, data, Util.both(cb, next)); + }); }, // CHANNEL ITERATION @@ -996,13 +1030,17 @@ module.exports.create = function (conf, cb) { getChannelSize: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - channelBytes(env, channelName, cb); + queue(channelName, function (next) { + channelBytes(env, channelName, Util.both(cb, next)); + }); }, // OTHER DATABASE FUNCTIONALITY // remove a particular channel from the cache closeChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - closeChannel(env, channelName, cb); + queue(channelName, function (next) { + closeChannel(env, channelName, Util.both(cb, next)); + }); }, // iterate over open channels and close any that are not active flushUnusedChannels: function (cb) { From 47fdf9de9f8d484b93a710927f1d65688af782b5 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 13 Jan 2020 10:32:31 -0500 Subject: [PATCH 03/53] add a note to fix some odd behaviour --- storage/file.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/file.js b/storage/file.js index 2a8802c8c..eca3d76be 100644 --- a/storage/file.js +++ b/storage/file.js @@ -759,6 +759,8 @@ var getChannel = function ( } if (env.openFiles >= env.openFileLimit) { + // FIXME warn if this is the case? + // alternatively use graceful-fs to handle lots of concurrent reads // if you're running out of open files, asynchronously clean up expired files // do it on a shorter timeframe, though (half of normal) setTimeout(function () { From 7072fe4fa4b6eebd7b693bcbce657bb22f3398d6 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 15 Jan 2020 13:32:52 -0500 Subject: [PATCH 04/53] implement and test a complex constrained scheduler --- lib/schedule.js | 173 +++++++++++++++++++++++++ scripts/tests/test-scheduler.js | 220 ++++++++++++++++++++++++++++++++ 2 files changed, 393 insertions(+) create mode 100644 lib/schedule.js create mode 100644 scripts/tests/test-scheduler.js diff --git a/lib/schedule.js b/lib/schedule.js new file mode 100644 index 000000000..1e93a2126 --- /dev/null +++ b/lib/schedule.js @@ -0,0 +1,173 @@ +var WriteQueue = require("./write-queue"); +var Util = require("./common-util"); + +/* This module provides implements a FIFO scheduler + which assumes the existence of three types of async tasks: + + 1. ordered tasks which must be executed sequentially + 2. unordered tasks which can be executed in parallel + 3. blocking tasks which must block the execution of all other tasks + + The scheduler assumes there will be many resources identified by strings, + and that the constraints described above will only apply in the context + of identical string ids. + + Many blocking tasks may be executed in parallel so long as they + concern resources identified by different ids. + +USAGE: + + const schedule = require("./schedule")(); + + // schedule two sequential tasks using the resource 'pewpew' + schedule.ordered('pewpew', function (next) { + appendToFile('beep\n', next); + }); + schedule.ordered('pewpew', function (next) { + appendToFile('boop\n', next); + }); + + // schedule a task that can happen whenever + schedule.unordered('pewpew', function (next) { + displayFileSize(next); + }); + + // schedule a blocking task which will wait + // until the all unordered tasks have completed before commencing + schedule.blocking('pewpew', function (next) { + deleteFile(next); + }); + + // this will be queued for after the blocking task + schedule.ordered('pewpew', function (next) { + appendFile('boom', next); + }); + +*/ + +// return a uid which is not already in a map +var unusedUid = function (set) { + var uid = Util.uid(); + if (set[uid]) { return unusedUid(); } + return uid; +}; + +// return an existing session, creating one if it does not already exist +var lookup = function (map, id) { + return (map[id] = map[id] || { + //blocking: [], + active: {}, + blocked: {}, + }); +}; + +var isEmpty = function (map) { + for (var key in map) { + if (map.hasOwnProperty(key)) { return false; } + } + return true; +}; + +// XXX enforce asynchrony everywhere +module.exports = function () { + // every scheduler instance has its own queue + var queue = WriteQueue(); + + // ordered tasks don't require any extra logic + var Ordered = function (id, task) { + queue(id, task); + }; + + // unordered and blocking tasks need a little extra state + var map = {}; + + // regular garbage collection keeps memory consumption low + var collectGarbage = function (id) { + // avoid using 'lookup' since it creates a session implicitly + var local = map[id]; + // bail out if no session + if (!local) { return; } + // bail out if there are blocking or active tasks + if (local.lock) { return; } + if (!isEmpty(local.active)) { return; } + // if there are no pending actions then delete the session + delete map[id]; + }; + + // unordered tasks run immediately if there are no blocking tasks scheduled + // or immediately after blocking tasks finish + var runImmediately = function (local, task) { + // set a flag in the map of active unordered tasks + // to prevent blocking tasks from running until you finish + var uid = unusedUid(local.active); + local.active[uid] = true; + + task(function () { + // remove the flag you set to indicate that your task completed + delete local.active[uid]; + // don't do anything if other unordered tasks are still running + if (!isEmpty(local.active)) { return; } + // bail out if there are no blocking tasks scheduled or ready + if (typeof(local.waiting) !== 'function') { + return void collectGarbage(); + } + local.waiting(); + }); + }; + + var runOnceUnblocked = function (local, task) { + var uid = unusedUid(local.blocked); + local.blocked[uid] = function () { + runImmediately(local, task); + }; + }; + + // 'unordered' tasks are scheduled to run in after the most recently received blocking task + // or immediately and in parallel if there are no blocking tasks scheduled. + var Unordered = function (id, task) { + var local = lookup(map, id); + if (local.lock) { return runOnceUnblocked(local, task); } + runImmediately(local, task); + }; + + var runBlocked = function (local) { + for (var task in local.blocked) { + runImmediately(local, local.blocked[task]); + } + }; + + // 'blocking' tasks must be run alone. + // They are queued alongside ordered tasks, + // and wait until any running 'unordered' tasks complete before commencing. + var Blocking = function (id, task) { + var local = lookup(map, id); + + queue(id, function (next) { + // start right away if there are no running unordered tasks + if (isEmpty(local.active)) { + local.lock = true; + return void task(function () { + delete local.lock; + runBlocked(local); + next(); + }); + } + // otherwise wait until the running tasks have completed + local.waiting = function () { + local.lock = true; + task(function () { + delete local.lock; + delete local.waiting; + runBlocked(local); + next(); + }); + }; + }); + }; + + return { + ordered: Ordered, + unordered: Unordered, + blocking: Blocking, + }; +}; diff --git a/scripts/tests/test-scheduler.js b/scripts/tests/test-scheduler.js new file mode 100644 index 000000000..6a076d5aa --- /dev/null +++ b/scripts/tests/test-scheduler.js @@ -0,0 +1,220 @@ +/* three types of actions: + * read + * write + * append + each of which take a random amount of time + +*/ +var Util = require("../../lib/common-util"); +var schedule = require("../../lib/schedule")(); +var nThen = require("nthen"); + +var rand = function (n) { + return Math.floor(Math.random() * n); +}; + +var rand_time = function () { + // between 51 and 151 + return rand(300) + 25; +}; + +var makeAction = function (type) { + var i = 0; + return function (time) { + var j = i++; + return function (next) { + console.log(" Beginning action: %s#%s", type, j); + setTimeout(function () { + console.log(" Completed action: %s#%s", type, j); + next(); + }, time); + return j; + }; + }; +}; + +var TYPES = ['WRITE', 'READ', 'APPEND']; +var chooseAction = function () { + var n = rand(100); + + if (n < 50) { return 'APPEND'; } + if (n < 90) { return 'READ'; } + return 'WRITE'; + + //return TYPES[rand(3)]; +}; + +var test = function (script, cb) { + var uid = Util.uid(); + + var TO_RUN = script.length; + var total_run = 0; + + var parallel = 0; + var last_run_ordered = -1; + //var i = 0; + + var ACTIONS = {}; + TYPES.forEach(function (type) { + ACTIONS[type] = makeAction(type); + }); + + nThen(function (w) { + setTimeout(w(), 3000); + // run scripted actions with assertions + script.forEach(function (scene) { + var type = scene[0]; + var time = typeof(scene[1]) === 'number'? scene[1]: rand_time(); + + var action = ACTIONS[type](time); + console.log("Queuing action of type: %s(%s)", type, time); + + var proceed = w(); + + switch (type) { + case 'APPEND': + return schedule.ordered(uid, w(function (next) { + parallel++; + var temp = action(function () { + parallel--; + total_run++; + proceed(); + next(); + }); + if (temp !== (last_run_ordered + 1)) { + throw new Error("out of order"); + } + last_run_ordered = temp; + })); + case 'WRITE': + return schedule.blocking(uid, w(function (next) { + parallel++; + action(function () { + parallel--; + total_run++; + proceed(); + next(); + }); + if (parallel > 1) { + console.log("parallelism === %s", parallel); + throw new Error("too much parallel"); + } + })); + case 'READ': + return schedule.unordered(uid, w(function (next) { + parallel++; + action(function () { + parallel--; + total_run++; + proceed(); + next(); + }); + })); + default: + throw new Error("wut"); + } + }); + }).nThen(function () { + // make assertions about the whole script + if (total_run !== TO_RUN) { + console.log("Ran %s / %s", total_run, TO_RUN); + throw new Error("skipped tasks"); + } + console.log("total_run === %s", total_run); + + cb(); + }); +}; + + +var randomScript = function () { + var len = rand(15) + 10; + var script = []; + while (len--) { + script.push([ + chooseAction(), + rand_time(), + ]); + } + return script; +}; + +var WRITE = function (t) { + return ['WRITE', t]; +}; +var READ = function (t) { + return ['READ', t]; +}; + +var APPEND = function (t) { + return ['APPEND', t]; +}; + +nThen(function (w) { + test([ + ['READ', 150], + ['APPEND', 200], + ['APPEND', 100], + ['READ', 350], + ['WRITE', 400], + ['APPEND', 275], + ['APPEND', 187], + ['WRITE', 330], + ['WRITE', 264], + ['WRITE', 256], + ], w(function () { + console.log("finished pre-scripted test\n"); + })); +}).nThen(function (w) { + test([ + WRITE(289), + APPEND(281), + READ(207), + WRITE(225), + READ(279), + WRITE(300), + READ(331), + APPEND(341), + APPEND(385), + READ(313), + WRITE(285), + READ(304), + APPEND(273), + APPEND(150), + WRITE(246), + READ(244), + WRITE(172), + APPEND(253), + READ(215), + READ(296), + APPEND(281), + APPEND(296), + WRITE(168), + ], w(function () { + console.log("finished 2nd pre-scripted test\n"); + })); +}).nThen(function () { + var totalTests = 50; + var randomTests = 1; + + var last = nThen(function () { + console.log("beginning randomized tests"); + }); + + var queueRandomTest = function (i) { + last = last.nThen(function (w) { + console.log("running random test script #%s\n", i); + test(randomScript(), w(function () { + console.log("finished random test #%s\n", i); + })); + }); + }; + + while (randomTests <=totalTests) { queueRandomTest(randomTests++); } + + last.nThen(function () { + console.log("finished %s random tests", totalTests); + }); +}); + + From 15ca855f221d07653e8b01ff7af0fa0488600471 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 15 Jan 2020 15:09:30 -0500 Subject: [PATCH 05/53] start using the scheduler for all relevant database methods ...and describe the rationale for using particular scheduler semantics --- storage/file.js | 143 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 114 insertions(+), 29 deletions(-) diff --git a/storage/file.js b/storage/file.js index eca3d76be..b2899619f 100644 --- a/storage/file.js +++ b/storage/file.js @@ -8,7 +8,7 @@ var nThen = require("nthen"); var Semaphore = require("saferphore"); var Util = require("../lib/common-util"); -const WriteQueue = require("../lib/write-queue"); +const Schedule = require("../lib/schedule"); const Readline = require("readline"); const ToPull = require('stream-to-pull-stream'); const Pull = require('pull-stream'); @@ -171,7 +171,6 @@ var readMessages = function (path, msgHandler, cb) { could have been amended */ var getChannelMetadata = function (Env, channelId, cb) { - // XXX queue var path = mkPath(Env, channelId); // gets metadata embedded in a file @@ -180,7 +179,6 @@ var getChannelMetadata = function (Env, channelId, cb) { // low level method for getting just the dedicated metadata channel var getDedicatedMetadata = function (env, channelId, handler, cb) { - // XXX queue var metadataPath = mkMetadataPath(env, channelId); readMessages(metadataPath, function (line) { if (!line) { return; } @@ -223,7 +221,6 @@ How to proceed */ - // XXX queue nThen(function (w) { // returns the first line of a channel, parsed... getChannelMetadata(env, channelId, w(function (err, data) { @@ -268,7 +265,6 @@ var writeMetadata = function (env, channelId, data, cb) { // TODO see if we could improve performance by using libnewline const NEWLINE_CHR = ('\n').charCodeAt(0); const mkBufferSplit = () => { - // XXX queue lock let remainder = null; return Pull((read) => { return (abort, cb) => { @@ -320,7 +316,6 @@ const mkOffsetCounter = () => { const readMessagesBin = (env, id, start, msgHandler, cb) => { const stream = Fs.createReadStream(mkPath(env, id), { start: start }); let keepReading = true; - // XXX queue lock Pull( ToPull.read(stream), mkBufferSplit(), @@ -846,7 +841,6 @@ var message = function (env, chanName, msg, cb) { // stream messages from a channel log var getMessages = function (env, chanName, handler, cb) { - // XXX queue lock getChannel(env, chanName, function (err, chan) { if (!chan) { cb(err); @@ -888,9 +882,26 @@ module.exports.create = function (conf, cb) { openFiles: 0, openFileLimit: conf.openFileLimit || 2048, }; - var queue = env.queue = WriteQueue(); var it; + /* our scheduler prioritizes and executes tasks with respect + to all other tasks invoked with an identical key + (typically the id of the concerned channel) + + it assumes that all tasks can be categorized into three types + + 1. unordered tasks such as streaming reads which can take + a long time to complete. + + 2. ordered tasks such as appending to a file which does not + take very long, but where priority is important. + + 3. blocking tasks such as rewriting a file where it would be + dangerous to perform any other task concurrently. + + */ + var schedule = env.schedule = Schedule(); + nThen(function (w) { // make sure the store's directory exists Fse.mkdirp(env.root, PERMISSIVE, w(function (err) { @@ -910,61 +921,112 @@ module.exports.create = function (conf, cb) { // write a new message to a log message: function (channelName, content, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { + schedule.ordered(channelName, function (next) { message(env, channelName, content, Util.both(cb, next)); }); }, // iterate over all the messages in a log getMessages: function (channelName, msgHandler, cb) { - // XXX queue lock if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - getMessages(env, channelName, msgHandler, cb); + schedule.unordered(channelName, function (next) { + getMessages(env, channelName, msgHandler, Util.both(cb, next)); + }); }, // NEWER IMPLEMENTATIONS OF THE SAME THING // write a new message to a log messageBin: (channelName, content, cb) => { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { + schedule.ordered(channelName, function (next) { messageBin(env, channelName, content, Util.both(cb, next)); }); }, // iterate over the messages in a log readMessagesBin: (channelName, start, asyncMsgHandler, cb) => { - // XXX queue lock if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - readMessagesBin(env, channelName, start, asyncMsgHandler, cb); +// XXX there is a race condition here +// historyKeeper reads the file to find the byte offset of the first interesting message +// then calls this function again to read from that point. +// If this task is in the queue already when the file is read again +// then that byte offset will have been invalidated +// and the resulting stream probably won't align with message boundaries. +// We can evict the cache in the callback but by that point it will be too late. +// Presumably we'll need to bury some of historyKeeper's logic into a filestore method +// in order to make index/read sequences atomic. +// Otherwise, we can add a new task type to the scheduler to take invalidation into account... +// either method introduces significant complexity. + schedule.unordered(channelName, function (next) { + readMessagesBin(env, channelName, start, asyncMsgHandler, Util.both(cb, next)); + }); }, // METHODS for deleting data // remove a channel and its associated metadata log if present removeChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// XXX there's another race condition here... +// when a remove and an append are scheduled in that order +// the remove will delete the channel's metadata (including its validateKey) +// then the append will recreate the channel and insert a message. +// clients that are connected to the channel via historyKeeper should be kicked out +// however, anyone that connects to that channel in the future will be able to read the +// signed message, but will not find its validate key... +// resulting in a junk/unusable document + schedule.ordered(channelName, function (next) { removeChannel(env, channelName, Util.both(cb, next)); }); }, // remove a channel and its associated metadata log from the archive directory removeArchivedChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { + schedule.ordered(channelName, function (next) { removeArchivedChannel(env, channelName, Util.both(cb, next)); }); }, // clear all data for a channel but preserve its metadata clearChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { + schedule.ordered(channelName, function (next) { clearChannel(env, channelName, Util.both(cb, next)); }); }, + trimChannel: function (channelName, hash, cb) { + // XXX ansuz + // XXX queue lock + /* block any reads from the metadata and log files + until this whole process has finished + close the file descriptor if it is open + derive temporary file paths for metadata and log buffers + compute metadata state and write to metadata buffer + scan through log file and begin copying lines to the log buffer + once you recognize the first line by the hash the user provided + archive the file and current metadata once both buffers are copied + move the metadata and log buffers into place + return the lock on reads + call back + + in case of an error, remove the buffer files + */ + schedule.blocking(channelName, function (next) { + cb("E_NOT_IMPLEMENTED"); + next(); + }); + }, // check if a channel exists in the database isChannelAvailable: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } // construct the path var filepath = mkPath(env, channelName); - queue(channelName, function (next) { +// (ansuz) I'm uncertain whether this task should be unordered or ordered. +// there's a round trip to the client (and possibly the user) before they decide +// to act on the information of whether there is already content present in this channel. +// so it's practically impossible to avoid race conditions where someone else creates +// some content before you. +// if that's the case, it's basically impossible that you'd generate the same signing key, +// and thus historykeeper should reject the signed messages of whoever loses the race. +// thus 'unordered' seems appropriate. + schedule.unordered(channelName, function (next) { channelExists(filepath, Util.both(cb, next)); }); }, @@ -973,21 +1035,30 @@ module.exports.create = function (conf, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } // construct the path var filepath = mkArchivePath(env, channelName); - queue(channelName, function (next) { +// as with the method above, somebody might remove, restore, or overwrite an archive +// in the time that it takes to answer this query and to execute whatever follows. +// since it's impossible to win the race every time let's just make this 'unordered' + schedule.unordered(channelName, function (next) { channelExists(filepath, Util.both(cb, next)); }); }, // move a channel from the database to the archive, along with its metadata archiveChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// again, the semantics around archiving and appending are really muddy. +// so I'm calling this 'unordered' again + schedule.unordered(channelName, function (next) { archiveChannel(env, channelName, Util.both(cb, next)); }); }, // restore a channel from the archive to the database, along with its metadata restoreArchivedChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// archive restoration will fail if either a file or its metadata exists in the live db. +// so I'm calling this 'ordered' to give writes a chance to flush out. +// accidental conflicts are extremely unlikely since clients check the status +// of a previously known channel before joining. + schedule.ordered(channelName, function (next) { unarchiveChannel(env, channelName, Util.both(cb, next)); }); }, @@ -996,14 +1067,17 @@ module.exports.create = function (conf, cb) { // fetch the metadata for a channel getChannelMetadata: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// The only thing that can invalid this method's results are channel archival, removal, or trimming. +// We want it to be fast, so let's make it unordered. + schedule.unordered(channelName, function (next) { getChannelMetadata(env, channelName, Util.both(cb, next)); }); }, // iterate over lines of metadata changes from a dedicated log readDedicatedMetadata: function (channelName, handler, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// Everything that modifies metadata also updates clients, so this can be 'unordered' + schedule.unordered(channelName, function (next) { getDedicatedMetadata(env, channelName, handler, Util.both(cb, next)); }); }, @@ -1011,13 +1085,16 @@ module.exports.create = function (conf, cb) { // iterate over multiple lines of metadata changes readChannelMetadata: function (channelName, handler, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - // XXX queue - readMetadata(env, channelName, handler, cb); +// same logic as 'readDedicatedMetadata + schedule.unordered(channelName, function (next) { + readMetadata(env, channelName, handler, Util.both(cb, next)); + }); }, // write a new line to a metadata log writeMetadata: function (channelName, data, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// metadata writes are fast and should be applied in order + schedule.ordered(channelName, function (next) { writeMetadata(env, channelName, data, Util.both(cb, next)); }); }, @@ -1032,7 +1109,9 @@ module.exports.create = function (conf, cb) { getChannelSize: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// this method should be really fast and it probably doesn't matter much +// if we get the size slightly before or after somebody writes a few hundred bytes to it. + schedule.ordered(channelName, function (next) { channelBytes(env, channelName, Util.both(cb, next)); }); }, @@ -1040,7 +1119,10 @@ module.exports.create = function (conf, cb) { // remove a particular channel from the cache closeChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// It is most likely the case that the channel is inactive if we are trying to close it, +// thus it doesn't make much difference whether it's ordered or not. +// In any case, it will be re-opened if anyone tries to write to it. + schedule.ordered(channelName, function (next) { closeChannel(env, channelName, Util.both(cb, next)); }); }, @@ -1050,7 +1132,10 @@ module.exports.create = function (conf, cb) { }, // write to a log file log: function (channelName, content, cb) { - message(env, channelName, content, cb); +// you probably want the events in your log to be in the correct order. + schedule.ordered(channelName, function (next) { + message(env, channelName, content, Util.both(cb, next)); + }); }, // shut down the database shutdown: function () { From ff40538ee7b2d8e033533d62be49341cdfb92425 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 15 Jan 2020 15:11:45 -0500 Subject: [PATCH 06/53] sketch our two new RPCs for trimming history --- rpc.js | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/rpc.js b/rpc.js index fcd85a390..8041cdcad 100644 --- a/rpc.js +++ b/rpc.js @@ -933,6 +933,16 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { }); }; +var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { + // XXX validate that the user sending the request owns the channel in question + // proceed to call Env.msgStore.trimChannel(channelId, hash, cb) if ok + // otherwise reject with INSUFFICIENT_PERMISSIONS + + // XXX if trimChannel calls back without an error + // you must also clear the channel's index from historyKeeper cache + cb("E_NOT_IMPLEMENTED"); +}; + /* Users should be able to clear their own pin log with an authenticated RPC */ var removePins = function (Env, safeKey, cb) { @@ -949,6 +959,11 @@ var removePins = function (Env, safeKey, cb) { }); }; +var trimPins = function (Env, safeKey, cb) { + // XXX trim to latest pin checkpoint + cb("NOT_IMPLEMENTED"); +}; + /* We assume that the server is secured against MitM attacks via HTTPS, and that malicious actors do not have code execution @@ -1313,9 +1328,11 @@ var isAuthenticatedCall = function (call) { 'OWNED_UPLOAD_COMPLETE', 'UPLOAD_CANCEL', 'EXPIRE_SESSION', + 'TRIM_OWNED_CHANNEL_HISTORY', 'CLEAR_OWNED_CHANNEL', 'REMOVE_OWNED_CHANNEL', 'REMOVE_PINS', + 'TRIM_PINS', 'WRITE_LOGIN_BLOCK', 'REMOVE_LOGIN_BLOCK', 'ADMIN', @@ -1635,11 +1652,21 @@ RPC.create = function ( if (e) { return void Respond(e); } Respond(void 0, "OK"); }); + case 'TRIM_OWNED_CHANNEL_HISTORY': + return void removeOwnedChannelHistory(Env, msg[1], publicKey, msg[2], function (e) { + if (e) { return void Respond(e); } + Respond(void 0, 'OK'); + }); case 'REMOVE_PINS': return void removePins(Env, safeKey, function (e) { if (e) { return void Respond(e); } Respond(void 0, "OK"); }); + case 'TRIM_PINS': + return void trimPins(Env, safeKey, function (e) { + if (e) { return void Respond(e); } + Respond(void 0, "OK"); + }); case 'UPLOAD': return void Env.blobStore.upload(safeKey, msg[1], function (e, len) { WARN(e, len); From 4418f6a113245cb009620c57902c9e4078900aa2 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 16 Jan 2020 14:24:07 -0500 Subject: [PATCH 07/53] tiny optimization which saves a little bit of memory usage for a little bit of time --- historyKeeper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/historyKeeper.js b/historyKeeper.js index 59a694e8c..5f9c48c93 100644 --- a/historyKeeper.js +++ b/historyKeeper.js @@ -284,9 +284,9 @@ module.exports.create = function (cfg) { const storeMessage = function (ctx, channel, msg, isCp, optionalMessageHash) { const id = channel.id; - const msgBin = Buffer.from(msg + '\n', 'utf8'); queueStorage(id, function (next) { + const msgBin = Buffer.from(msg + '\n', 'utf8'); // Store the message first, and update the index only once it's stored. // store.messageBin can be async so updating the index first may // result in a wrong cpIndex From 8c5c643a25ff2b3e7d90e234d254af329926ccec Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 16 Jan 2020 14:25:00 -0500 Subject: [PATCH 08/53] finally get around to reorganizing the messiest part of history keeper --- historyKeeper.js | 416 ++++++++++++++++++++++++----------------------- 1 file changed, 213 insertions(+), 203 deletions(-) diff --git a/historyKeeper.js b/historyKeeper.js index 5f9c48c93..fe16a204c 100644 --- a/historyKeeper.js +++ b/historyKeeper.js @@ -730,227 +730,204 @@ module.exports.create = function (cfg) { } }; - /* onDirectMessage - * exported for use by the netflux-server - * parses and handles all direct messages directed to the history keeper - * check if it's expired and execute all the associated side-effects - * routes queries to the appropriate handlers - * GET_HISTORY - * GET_HISTORY_RANGE - * GET_FULL_HISTORY - * RPC - * if the rpc has special hooks that the history keeper needs to be aware of... - * execute them here... - - */ - const onDirectMessage = function (ctx, seq, user, json) { - let parsed; - let channelName; - - Log.silly('HK_MESSAGE', json); - - try { - parsed = JSON.parse(json[2]); - } catch (err) { - Log.error("HK_PARSE_CLIENT_MESSAGE", json); - return; - } - - // If the requested history is for an expired channel, abort - // Note the if we don't have the keys for that channel in metadata_cache, we'll - // have to abort later (once we know the expiration time) - if (checkExpired(ctx, parsed[1])) { return; } - - if (parsed[0] === 'GET_HISTORY') { - // parsed[1] is the channel id - // parsed[2] is a validation key or an object containing metadata (optionnal) - // parsed[3] is the last known hash (optionnal) - sendMsg(ctx, user, [seq, 'ACK']); - channelName = parsed[1]; - var config = parsed[2]; - var metadata = {}; - var lastKnownHash; - - // clients can optionally pass a map of attributes - // if the channel already exists this map will be ignored - // otherwise it will be stored as the initial metadata state for the channel - if (config && typeof config === "object" && !Array.isArray(parsed[2])) { - lastKnownHash = config.lastKnownHash; - metadata = config.metadata || {}; - if (metadata.expire) { - metadata.expire = +metadata.expire * 1000 + (+new Date()); - } - } - metadata.channel = channelName; - metadata.created = +new Date(); - - // if the user sends us an invalid key, we won't be able to validate their messages - // so they'll never get written to the log anyway. Let's just drop their message - // on the floor instead of doing a bunch of extra work - // TODO send them an error message so they know something is wrong - if (metadata.validateKey && !isValidValidateKeyString(metadata.validateKey)) { - return void Log.error('HK_INVALID_KEY', metadata.validateKey); + const handleGetHistory = function (ctx, seq, user, parsed) { + // parsed[1] is the channel id + // parsed[2] is a validation key or an object containing metadata (optionnal) + // parsed[3] is the last known hash (optionnal) + sendMsg(ctx, user, [seq, 'ACK']); + var channelName = parsed[1]; + var config = parsed[2]; + var metadata = {}; + var lastKnownHash; + + // clients can optionally pass a map of attributes + // if the channel already exists this map will be ignored + // otherwise it will be stored as the initial metadata state for the channel + if (config && typeof config === "object" && !Array.isArray(parsed[2])) { + lastKnownHash = config.lastKnownHash; + metadata = config.metadata || {}; + if (metadata.expire) { + metadata.expire = +metadata.expire * 1000 + (+new Date()); } + } + metadata.channel = channelName; + metadata.created = +new Date(); + + // if the user sends us an invalid key, we won't be able to validate their messages + // so they'll never get written to the log anyway. Let's just drop their message + // on the floor instead of doing a bunch of extra work + // TODO send them an error message so they know something is wrong + if (metadata.validateKey && !isValidValidateKeyString(metadata.validateKey)) { + return void Log.error('HK_INVALID_KEY', metadata.validateKey); + } - nThen(function (waitFor) { - var w = waitFor(); + nThen(function (waitFor) { + var w = waitFor(); - /* unless this is a young channel, we will serve all messages from an offset - this will not include the channel metadata, so we need to explicitly fetch that. - unfortunately, we can't just serve it blindly, since then young channels will - send the metadata twice, so let's do a quick check of what we're going to serve... + /* unless this is a young channel, we will serve all messages from an offset + this will not include the channel metadata, so we need to explicitly fetch that. + unfortunately, we can't just serve it blindly, since then young channels will + send the metadata twice, so let's do a quick check of what we're going to serve... + */ + getIndex(ctx, channelName, waitFor((err, index) => { + /* if there's an error here, it should be encountered + and handled by the next nThen block. + so, let's just fall through... */ - getIndex(ctx, channelName, waitFor((err, index) => { - /* if there's an error here, it should be encountered - and handled by the next nThen block. - so, let's just fall through... - */ - if (err) { return w(); } - - - // it's possible that the channel doesn't have metadata - // but in that case there's no point in checking if the channel expired - // or in trying to send metadata, so just skip this block - if (!index || !index.metadata) { return void w(); } - // And then check if the channel is expired. If it is, send the error and abort - // FIXME this is hard to read because 'checkExpired' has side effects - if (checkExpired(ctx, channelName)) { return void waitFor.abort(); } - // always send metadata with GET_HISTORY requests - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); - })); - }).nThen(() => { - let msgCount = 0; - - // TODO compute lastKnownHash in a manner such that it will always skip past the metadata line? - getHistoryAsync(ctx, channelName, lastKnownHash, false, (msg, readMore) => { - if (!msg) { return; } - msgCount++; - // avoid sending the metadata message a second time - if (isMetadataMessage(msg) && metadata_cache[channelName]) { return readMore(); } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)], readMore); - }, (err) => { - if (err && err.code !== 'ENOENT') { - if (err.message !== 'EINVAL') { Log.error("HK_GET_HISTORY", err); } - const parsedMsg = {error:err.message, channel: channelName}; - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); - return; - } - - const chan = ctx.channels[channelName]; + if (err) { return w(); } + + + // it's possible that the channel doesn't have metadata + // but in that case there's no point in checking if the channel expired + // or in trying to send metadata, so just skip this block + if (!index || !index.metadata) { return void w(); } + // And then check if the channel is expired. If it is, send the error and abort + // FIXME this is hard to read because 'checkExpired' has side effects + if (checkExpired(ctx, channelName)) { return void waitFor.abort(); } + // always send metadata with GET_HISTORY requests + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); + })); + }).nThen(() => { + let msgCount = 0; - if (msgCount === 0 && !metadata_cache[channelName] && chan && chan.indexOf(user) > -1) { - metadata_cache[channelName] = metadata; + // TODO compute lastKnownHash in a manner such that it will always skip past the metadata line? + getHistoryAsync(ctx, channelName, lastKnownHash, false, (msg, readMore) => { + if (!msg) { return; } + msgCount++; + // avoid sending the metadata message a second time + if (isMetadataMessage(msg) && metadata_cache[channelName]) { return readMore(); } + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)], readMore); + }, (err) => { + if (err && err.code !== 'ENOENT') { + if (err.message !== 'EINVAL') { Log.error("HK_GET_HISTORY", err); } + const parsedMsg = {error:err.message, channel: channelName}; + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + return; + } - // the index will have already been constructed and cached at this point - // but it will not have detected any metadata because it hasn't been written yet - // this means that the cache starts off as invalid, so we have to correct it - if (chan && chan.index) { chan.index.metadata = metadata; } + const chan = ctx.channels[channelName]; + + if (msgCount === 0 && !metadata_cache[channelName] && chan && chan.indexOf(user) > -1) { + metadata_cache[channelName] = metadata; + + // the index will have already been constructed and cached at this point + // but it will not have detected any metadata because it hasn't been written yet + // this means that the cache starts off as invalid, so we have to correct it + if (chan && chan.index) { chan.index.metadata = metadata; } + + // new channels will always have their metadata written to a dedicated metadata log + // but any lines after the first which are not amendments in a particular format will be ignored. + // Thus we should be safe from race conditions here if just write metadata to the log as below... + // TODO validate this logic + // otherwise maybe we need to check that the metadata log is empty as well + store.writeMetadata(channelName, JSON.stringify(metadata), function (err) { + if (err) { + // FIXME tell the user that there was a channel error? + return void Log.error('HK_WRITE_METADATA', { + channel: channelName, + error: err, + }); + } + }); - // new channels will always have their metadata written to a dedicated metadata log - // but any lines after the first which are not amendments in a particular format will be ignored. - // Thus we should be safe from race conditions here if just write metadata to the log as below... - // TODO validate this logic - // otherwise maybe we need to check that the metadata log is empty as well - store.writeMetadata(channelName, JSON.stringify(metadata), function (err) { + // write tasks + if(tasks && metadata.expire && typeof(metadata.expire) === 'number') { + // the fun part... + // the user has said they want this pad to expire at some point + tasks.write(metadata.expire, "EXPIRE", [ channelName ], function (err) { if (err) { - // FIXME tell the user that there was a channel error? - return void Log.error('HK_WRITE_METADATA', { - channel: channelName, - error: err, - }); + // if there is an error, we don't want to crash the whole server... + // just log it, and if there's a problem you'll be able to fix it + // at a later date with the provided information + Log.error('HK_CREATE_EXPIRE_TASK', err); + Log.info('HK_INVALID_EXPIRE_TASK', JSON.stringify([metadata.expire, 'EXPIRE', channelName])); } }); - - // write tasks - if(tasks && metadata.expire && typeof(metadata.expire) === 'number') { - // the fun part... - // the user has said they want this pad to expire at some point - tasks.write(metadata.expire, "EXPIRE", [ channelName ], function (err) { - if (err) { - // if there is an error, we don't want to crash the whole server... - // just log it, and if there's a problem you'll be able to fix it - // at a later date with the provided information - Log.error('HK_CREATE_EXPIRE_TASK', err); - Log.info('HK_INVALID_EXPIRE_TASK', JSON.stringify([metadata.expire, 'EXPIRE', channelName])); - } - }); - } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(metadata)]); } + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(metadata)]); + } - // End of history message: - let parsedMsg = {state: 1, channel: channelName}; - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); - }); + // End of history message: + let parsedMsg = {state: 1, channel: channelName}; + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); }); - } else if (parsed[0] === 'GET_HISTORY_RANGE') { - channelName = parsed[1]; - var map = parsed[2]; - if (!(map && typeof(map) === 'object')) { - return void sendMsg(ctx, user, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]); - } + }); + }; - var oldestKnownHash = map.from; - var desiredMessages = map.count; - var desiredCheckpoint = map.cpCount; - var txid = map.txid; - if (typeof(desiredMessages) !== 'number' && typeof(desiredCheckpoint) !== 'number') { - return void sendMsg(ctx, user, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]); - } + const handleGetHistoryRange = function (ctx, seq, user, parsed) { + var channelName = parsed[1]; + var map = parsed[2]; + if (!(map && typeof(map) === 'object')) { + return void sendMsg(ctx, user, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]); + } - if (!txid) { - return void sendMsg(ctx, user, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]); - } + var oldestKnownHash = map.from; + var desiredMessages = map.count; + var desiredCheckpoint = map.cpCount; + var txid = map.txid; + if (typeof(desiredMessages) !== 'number' && typeof(desiredCheckpoint) !== 'number') { + return void sendMsg(ctx, user, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]); + } - sendMsg(ctx, user, [seq, 'ACK']); - return void getOlderHistory(channelName, oldestKnownHash, function (messages) { - var toSend = []; - if (typeof (desiredMessages) === "number") { - toSend = messages.slice(-desiredMessages); - } else { - let cpCount = 0; - for (var i = messages.length - 1; i >= 0; i--) { - if (/^cp\|/.test(messages[i][4]) && i !== (messages.length - 1)) { - cpCount++; - } - toSend.unshift(messages[i]); - if (cpCount >= desiredCheckpoint) { break; } + if (!txid) { + return void sendMsg(ctx, user, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]); + } + + sendMsg(ctx, user, [seq, 'ACK']); + return void getOlderHistory(channelName, oldestKnownHash, function (messages) { + var toSend = []; + if (typeof (desiredMessages) === "number") { + toSend = messages.slice(-desiredMessages); + } else { + let cpCount = 0; + for (var i = messages.length - 1; i >= 0; i--) { + if (/^cp\|/.test(messages[i][4]) && i !== (messages.length - 1)) { + cpCount++; } + toSend.unshift(messages[i]); + if (cpCount >= desiredCheckpoint) { break; } } - toSend.forEach(function (msg) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, - JSON.stringify(['HISTORY_RANGE', txid, msg])]); - }); - + } + toSend.forEach(function (msg) { sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, - JSON.stringify(['HISTORY_RANGE_END', txid, channelName]) - ]); - }); - } else if (parsed[0] === 'GET_FULL_HISTORY') { - // parsed[1] is the channel id - // parsed[2] is a validation key (optionnal) - // parsed[3] is the last known hash (optionnal) - sendMsg(ctx, user, [seq, 'ACK']); - - // FIXME should we send metadata here too? - // none of the clientside code which uses this API needs metadata, but it won't hurt to send it (2019-08-22) - getHistoryAsync(ctx, parsed[1], -1, false, (msg, readMore) => { - if (!msg) { return; } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['FULL_HISTORY', msg])], readMore); - }, (err) => { - let parsedMsg = ['FULL_HISTORY_END', parsed[1]]; - if (err) { - Log.error('HK_GET_FULL_HISTORY', err.stack); - parsedMsg = ['ERROR', parsed[1], err.message]; - } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + JSON.stringify(['HISTORY_RANGE', txid, msg])]); }); - } else if (rpc) { - /* RPC Calls... */ - var rpc_call = parsed.slice(1); - sendMsg(ctx, user, [seq, 'ACK']); - try { + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, + JSON.stringify(['HISTORY_RANGE_END', txid, channelName]) + ]); + }); + }; + + const handleGetFullHistory = function (ctx, seq, user, parsed) { + // parsed[1] is the channel id + // parsed[2] is a validation key (optionnal) + // parsed[3] is the last known hash (optionnal) + sendMsg(ctx, user, [seq, 'ACK']); + + // FIXME should we send metadata here too? + // none of the clientside code which uses this API needs metadata, but it won't hurt to send it (2019-08-22) + return void getHistoryAsync(ctx, parsed[1], -1, false, (msg, readMore) => { + if (!msg) { return; } + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['FULL_HISTORY', msg])], readMore); + }, (err) => { + let parsedMsg = ['FULL_HISTORY_END', parsed[1]]; + if (err) { + Log.error('HK_GET_FULL_HISTORY', err.stack); + parsedMsg = ['ERROR', parsed[1], err.message]; + } + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + }); + }; + + const handleRPC = function (ctx, seq, user, parsed) { + if (typeof(rpc) !== 'function') { return; } + + /* RPC Calls... */ + var rpc_call = parsed.slice(1); + + sendMsg(ctx, user, [seq, 'ACK']); + try { // slice off the sequence number and pass in the rest of the message rpc(ctx, rpc_call, function (err, output) { if (err) { @@ -992,10 +969,43 @@ module.exports.create = function (cfg) { // finally, send a response to the client that sent the RPC sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0]].concat(output))]); }); - } catch (e) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]); - } + } catch (e) { + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]); + } + }; + + /* onDirectMessage + * exported for use by the netflux-server + * parses and handles all direct messages directed to the history keeper + * check if it's expired and execute all the associated side-effects + * routes queries to the appropriate handlers + */ + const onDirectMessage = function (ctx, seq, user, json) { + Log.silly('HK_MESSAGE', json); + + let parsed; + try { + parsed = JSON.parse(json[2]); + } catch (err) { + Log.error("HK_PARSE_CLIENT_MESSAGE", json); + return; + } + + // If the requested history is for an expired channel, abort + // Note the if we don't have the keys for that channel in metadata_cache, we'll + // have to abort later (once we know the expiration time) + if (checkExpired(ctx, parsed[1])) { return; } + + if (parsed[0] === 'GET_HISTORY') { + return void handleGetHistory(ctx, seq, user, parsed); + } + if (parsed[0] === 'GET_HISTORY_RANGE') { + return void handleGetHistoryRange(ctx, seq, user, parsed); + } + if (parsed[0] === 'GET_FULL_HISTORY') { + return void handleGetFullHistory(ctx, seq, user, parsed); } + return void handleRPC(ctx, seq, user, parsed); }; return { From b585dd998dc255126128d6fe4bdc73a0409d6ca0 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 13:27:12 -0500 Subject: [PATCH 09/53] throw in a little asynchrony --- lib/schedule.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/schedule.js b/lib/schedule.js index 1e93a2126..1fdef8cce 100644 --- a/lib/schedule.js +++ b/lib/schedule.js @@ -68,7 +68,6 @@ var isEmpty = function (map) { return true; }; -// XXX enforce asynchrony everywhere module.exports = function () { // every scheduler instance has its own queue var queue = WriteQueue(); @@ -111,7 +110,7 @@ module.exports = function () { if (typeof(local.waiting) !== 'function') { return void collectGarbage(); } - local.waiting(); + setTimeout(local.waiting); }); }; From 75f1f8c40b63af8f3cee1113e815e4ea06780cf5 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 13:56:16 -0500 Subject: [PATCH 10/53] refactor some methods that will be used --- rpc.js | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/rpc.js b/rpc.js index 8041cdcad..0a2c5b17f 100644 --- a/rpc.js +++ b/rpc.js @@ -278,6 +278,24 @@ var getMetadata = function (Env, channel, cb) { }); }; +// E_NO_OWNERS +var hasOwners = function (metadata) { + return Boolean(metadata && Array.isArray(metadata.owners)); +}; + +var hasPendingOwners = function (metadata) { + return Boolean(metadata && Array.isArray(metadata.pending_owners)); +}; + +// INSUFFICIENT_PERMISSIONS +var isOwner = function (metadata, unsafeKey) { + return metadata.owners.indexOf(unsafeKey) !== -1; +}; + +var isPendingOwner = function (metadata, unsafeKey) { + return metadata.pending_owners.indexOf(unsafeKey) !== -1; +}; + /* setMetadata - write a new line to the metadata log if a valid command is provided - data is an object: { @@ -300,16 +318,16 @@ var setMetadata = function (Env, data, unsafeKey, cb) { cb(err); return void next(); } - if (!(metadata && Array.isArray(metadata.owners))) { + if (!hasOwners(metadata)) { cb('E_NO_OWNERS'); return void next(); } // Confirm that the channel is owned by the user in question - // or the user is accepting a pending ownerhsip offer - if (metadata.pending_owners && Array.isArray(metadata.pending_owners) && - metadata.pending_owners.indexOf(unsafeKey) !== -1 && - metadata.owners.indexOf(unsafeKey) === -1) { + // or the user is accepting a pending ownership offer + if (hasPendingOwners(metadata) && + isPendingOwner(metadata, unsafeKey) && + !isOwner(metadata, unsafeKey)) { // If you are a pending owner, make sure you can only add yourelf as an owner if ((command !== 'ADD_OWNERS' && command !== 'RM_PENDING_OWNERS') @@ -319,8 +337,8 @@ var setMetadata = function (Env, data, unsafeKey, cb) { cb('INSUFFICIENT_PERMISSIONS'); return void next(); } - - } else if (metadata.owners.indexOf(unsafeKey) === -1) { + // XXX wacky fallthrough is hard to read + } else if (!isOwner(metadata, unsafeKey)) { cb('INSUFFICIENT_PERMISSIONS'); return void next(); } @@ -817,12 +835,11 @@ var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } - if (!(metadata && Array.isArray(metadata.owners))) { return void cb('E_NO_OWNERS'); } + if (!hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } // Confirm that the channel is owned by the user in question - if (metadata.owners.indexOf(unsafeKey) === -1) { + if (!isOwner(metadata, unsafeKey)) { return void cb('INSUFFICIENT_PERMISSIONS'); } - // FIXME COLDSTORAGE return void Env.msgStore.clearChannel(channelId, function (e) { cb(e); }); @@ -905,8 +922,8 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } - if (!(metadata && Array.isArray(metadata.owners))) { return void cb('E_NO_OWNERS'); } - if (metadata.owners.indexOf(unsafeKey) === -1) { + if (!hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } + if (!isOwner(metadata, unsafeKey)) { return void cb('INSUFFICIENT_PERMISSIONS'); } // if the admin has configured data retention... From 47290fca1e60fc254d52214c2343f6772c2dec2f Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 14:10:20 -0500 Subject: [PATCH 11/53] leave some notes about something that was tricky to read --- rpc.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/rpc.js b/rpc.js index 0a2c5b17f..69eb9313f 100644 --- a/rpc.js +++ b/rpc.js @@ -323,6 +323,14 @@ var setMetadata = function (Env, data, unsafeKey, cb) { return void next(); } + // if you are a pending owner and not an owner + // you can either ADD_OWNERS, or RM_PENDING_OWNERS + // and you should only be able to add yourself as an owner + // everything else should be rejected + // else if you are not an owner + // you should be rejected + // else write the command + // Confirm that the channel is owned by the user in question // or the user is accepting a pending ownership offer if (hasPendingOwners(metadata) && @@ -337,7 +345,9 @@ var setMetadata = function (Env, data, unsafeKey, cb) { cb('INSUFFICIENT_PERMISSIONS'); return void next(); } - // XXX wacky fallthrough is hard to read + // FIXME wacky fallthrough is hard to read + // we could pass this off to a writeMetadataCommand function + // and make the flow easier to follow } else if (!isOwner(metadata, unsafeKey)) { cb('INSUFFICIENT_PERMISSIONS'); return void next(); From 88dcadcb1b561bfc6297b7bb7aad20a3f785e925 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 14:31:45 -0500 Subject: [PATCH 12/53] sketch out trimHistory interface --- rpc.js | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/rpc.js b/rpc.js index 69eb9313f..f690b2bf1 100644 --- a/rpc.js +++ b/rpc.js @@ -961,13 +961,27 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { }; var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { - // XXX validate that the user sending the request owns the channel in question - // proceed to call Env.msgStore.trimChannel(channelId, hash, cb) if ok - // otherwise reject with INSUFFICIENT_PERMISSIONS + nThen(function (w) { + getMetadata(Env, channelId, w(function (err, metadata) { + if (err) { return void cb(err); } + if (!hasOwners(metadata)) { + w.abort(); + return void cb('E_NO_OWNERS'); + } + if (!isOwner(metadata, unsafeKey)) { + w.abort(); + return void cb("INSUFFICIENT_PERMISSIONS"); + } + // else fall through to the next block + })); + }).nThen(function () { + Env.msgStore.trimChannel(channelId, hash, function (err) { + if (err) { return void cb(err); } + - // XXX if trimChannel calls back without an error - // you must also clear the channel's index from historyKeeper cache - cb("E_NOT_IMPLEMENTED"); + // XXX you must also clear the channel's index from historyKeeper cache + }); + }); }; /* Users should be able to clear their own pin log with an authenticated RPC From 873a7c7c84b01e77670a821e0712b285f543eba9 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 14:32:10 -0500 Subject: [PATCH 13/53] remove some flow annotations --- rpc.js | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/rpc.js b/rpc.js index f690b2bf1..948977bbd 100644 --- a/rpc.js +++ b/rpc.js @@ -1411,24 +1411,7 @@ var upload_status = function (Env, safeKey, filesize, _cb) { // FIXME FILES }); }; -/*:: -const flow_Config = require('./config.example.js'); -type Config_t = typeof(flow_Config); -import type { ChainPadServer_Storage_t } from './storage/file.js' -type NetfluxWebsocketSrvContext_t = { - store: ChainPadServer_Storage_t, - getHistoryOffset: ( - ctx: NetfluxWebsocketSrvContext_t, - channelName: string, - lastKnownHash: ?string, - cb: (err: ?Error, offset: ?number)=>void - )=>void -}; -*/ -RPC.create = function ( - config /*:Config_t*/, - cb /*:(?Error, ?Function)=>void*/ -) { +RPC.create = function (config, cb) { Log = config.log; // load pin-store... @@ -1445,7 +1428,7 @@ RPC.create = function ( Sessions: {}, paths: {}, msgStore: config.store, - pinStore: (undefined /*:any*/), + pinStore: undefined, pinnedPads: {}, evPinnedPadsReady: mkEvent(true), limits: {}, @@ -1782,11 +1765,7 @@ RPC.create = function ( handleMessage(true); }; - var rpc = function ( - ctx /*:NetfluxWebsocketSrvContext_t*/, - data /*:Array>*/, - respond /*:(?string, ?Array)=>void*/) - { + var rpc = function (ctx, data, respond) { try { return rpc0(ctx, data, respond); } catch (e) { From c14612528394d67fad427907d2f956a979181dba Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 14:32:32 -0500 Subject: [PATCH 14/53] sketch out a method for graceful shutdown --- rpc.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/rpc.js b/rpc.js index 948977bbd..1790682ec 100644 --- a/rpc.js +++ b/rpc.js @@ -1318,6 +1318,16 @@ var getActiveSessions = function (Env, ctx, cb) { cb (void 0, [total, ips.length]); }; +var shutdown = function (Env, ctx, cb) { + return void cb('E_NOT_IMPLEMENTED'); + //clearInterval(Env.sessionExpirationInterval); + // XXX set a flag to prevent incoming database writes + // XXX disconnect all users and reject new connections + // XXX wait until all pending writes are complete + // then process.exit(0); + // and allow system functionality to restart the server +}; + var adminCommand = function (Env, ctx, publicKey, config, data, cb) { var admins = Env.admins; if (admins.indexOf(publicKey) === -1) { @@ -1336,6 +1346,8 @@ var adminCommand = function (Env, ctx, publicKey, config, data, cb) { case 'FLUSH_CACHE': config.flushCache(); return cb(void 0, true); + case 'SHUTDOWN': + return shutdown(Env, ctx, cb); default: return cb('UNHANDLED_ADMIN_COMMAND'); } @@ -1806,7 +1818,8 @@ RPC.create = function (config, cb) { }).nThen(function () { cb(void 0, rpc); // expire old sessions once per minute - setInterval(function () { + // XXX allow for graceful shutdown + Env.sessionExpirationInterval = setInterval(function () { expireSessions(Sessions); }, SESSION_EXPIRATION_TIME); }); From c4194117a7896f7d968c745dbfaf7cb867b28d21 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 14:33:00 -0500 Subject: [PATCH 15/53] ever so slightly faster direct message handler --- historyKeeper.js | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/historyKeeper.js b/historyKeeper.js index fe16a204c..2cb0f98ec 100644 --- a/historyKeeper.js +++ b/historyKeeper.js @@ -974,6 +974,12 @@ module.exports.create = function (cfg) { } }; + const directMessageCommands = { + GET_HISTORY: handleGetHistory, + GET_HISTORY_RANGE: handleGetHistoryRange, + GET_FULL_HISTORY: handleGetFullHistory, + }; + /* onDirectMessage * exported for use by the netflux-server * parses and handles all direct messages directed to the history keeper @@ -996,16 +1002,11 @@ module.exports.create = function (cfg) { // have to abort later (once we know the expiration time) if (checkExpired(ctx, parsed[1])) { return; } - if (parsed[0] === 'GET_HISTORY') { - return void handleGetHistory(ctx, seq, user, parsed); - } - if (parsed[0] === 'GET_HISTORY_RANGE') { - return void handleGetHistoryRange(ctx, seq, user, parsed); - } - if (parsed[0] === 'GET_FULL_HISTORY') { - return void handleGetFullHistory(ctx, seq, user, parsed); - } - return void handleRPC(ctx, seq, user, parsed); + // look up the appropriate command in the map of commands or fall back to RPC + var command = directMessageCommands[parsed[0]] || handleRPC; + + // run the command with the standard function signature + command(ctx, seq, user, parsed); }; return { From f45de2b52f81465c2d3293512216aaf06f38f178 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 14:47:55 -0500 Subject: [PATCH 16/53] move some server deps from repo root to lib/ --- historyKeeper.js => lib/historyKeeper.js | 8 ++++---- rpc.js => lib/rpc.js | 22 +++++++++++++--------- server.js | 4 ++-- 3 files changed, 19 insertions(+), 15 deletions(-) rename historyKeeper.js => lib/historyKeeper.js (99%) rename rpc.js => lib/rpc.js (98%) diff --git a/historyKeeper.js b/lib/historyKeeper.js similarity index 99% rename from historyKeeper.js rename to lib/historyKeeper.js index 2cb0f98ec..414d064c3 100644 --- a/historyKeeper.js +++ b/lib/historyKeeper.js @@ -5,10 +5,10 @@ const nThen = require('nthen'); const Nacl = require('tweetnacl/nacl-fast'); const Crypto = require('crypto'); -const Once = require("./lib/once"); -const Meta = require("./lib/metadata"); -const WriteQueue = require("./lib/write-queue"); -const BatchRead = require("./lib/batch-read"); +const Once = require("./once"); +const Meta = require("./metadata"); +const WriteQueue = require("./write-queue"); +const BatchRead = require("./batch-read"); let Log; const now = function () { return (new Date()).getTime(); }; diff --git a/rpc.js b/lib/rpc.js similarity index 98% rename from rpc.js rename to lib/rpc.js index 1790682ec..dfe27302d 100644 --- a/rpc.js +++ b/lib/rpc.js @@ -11,25 +11,25 @@ var Fs = require("fs"); var Fse = require("fs-extra"); var Path = require("path"); var Https = require("https"); -const Package = require('./package.json'); -const Pinned = require('./scripts/pinned'); +const Package = require('../package.json'); +const Pinned = require('../scripts/pinned'); const Saferphore = require("saferphore"); const nThen = require("nthen"); const getFolderSize = require("get-folder-size"); -const Pins = require("./lib/pins"); -const Meta = require("./lib/metadata"); -const WriteQueue = require("./lib/write-queue"); -const BatchRead = require("./lib/batch-read"); +const Pins = require("./pins"); +const Meta = require("./metadata"); +const WriteQueue = require("./write-queue"); +const BatchRead = require("./batch-read"); -const Util = require("./lib/common-util"); +const Util = require("./common-util"); const escapeKeyCharacters = Util.escapeKeyCharacters; const unescapeKeyCharacters = Util.unescapeKeyCharacters; const mkEvent = Util.mkEvent; var RPC = module.exports; -var Store = require("./storage/file"); -var BlobStore = require("./storage/blob"); +var Store = require("../storage/file"); +var BlobStore = require("../storage/blob"); var DEFAULT_LIMIT = 50 * 1024 * 1024; var SESSION_EXPIRATION_TIME = 60 * 1000; @@ -611,6 +611,9 @@ var updateLimits = function (Env, config, publicKey, cb /*:(?string, ?any[])=>vo req.end(body); }; +// XXX it's possible for this to respond before the server has had a chance +// to fetch the limits. Maybe we should respond with an error... +// or wait until we actually know the limits before responding var getLimit = function (Env, publicKey, cb) { var unescapedKey = unescapeKeyCharacters(publicKey); var limit = Env.limits[unescapedKey]; @@ -1445,6 +1448,7 @@ RPC.create = function (config, cb) { evPinnedPadsReady: mkEvent(true), limits: {}, admins: [], + sessionExpirationInterval: undefined, }; try { diff --git a/server.js b/server.js index 399eb1442..fec59a7ea 100644 --- a/server.js +++ b/server.js @@ -264,7 +264,7 @@ var nt = nThen(function (w) { }, 1000 * 60 * 5); // run every five minutes })); }).nThen(function (w) { - require("./rpc").create(config, w(function (e, _rpc) { + require("./lib/rpc").create(config, w(function (e, _rpc) { if (e) { w.abort(); throw e; @@ -272,7 +272,7 @@ var nt = nThen(function (w) { rpc = _rpc; })); }).nThen(function () { - var HK = require('./historyKeeper.js'); + var HK = require('./lib/historyKeeper.js'); var hkConfig = { tasks: config.tasks, rpc: rpc, From c388641479128303363d8a4247f64230c08a7264 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 15:13:19 -0500 Subject: [PATCH 17/53] drop support for 'retainData' configuration --- config/config.example.js | 21 +++++---------- lib/historyKeeper.js | 16 +++-------- lib/rpc.js | 57 ++++++--------------------------------- scripts/evict-inactive.js | 50 ++++------------------------------ server.js | 1 - storage/file.js | 4 --- storage/tasks.js | 17 ------------ 7 files changed, 22 insertions(+), 144 deletions(-) diff --git a/config/config.example.js b/config/config.example.js index 9981c0626..90e96a66a 100644 --- a/config/config.example.js +++ b/config/config.example.js @@ -231,26 +231,17 @@ module.exports = { */ inactiveTime: 90, // days - /* CryptPad can be configured to remove inactive data which has not been pinned. - * Deletion of data is always risky and as an operator you have the choice to - * archive data instead of deleting it outright. Set this value to true if - * you want your server to archive files and false if you want to keep using - * the old behaviour of simply removing files. + /* CryptPad archives some data instead of deleting it outright. + * This archived data still takes up space and so you'll probably still want to + * remove these files after a brief period. + * + * cryptpad/scripts/evict-inactive.js is intended to be run daily + * from a crontab or similar scheduling service. * - * WARNING: this is not implemented universally, so at the moment this will - * only apply to the removal of 'channels' due to inactivity. - */ - retainData: true, - - /* As described above, CryptPad offers the ability to archive some data - * instead of deleting it outright. This archived data still takes up space - * and so you'll probably still want to remove these files after a brief period. * The intent with this feature is to provide a safety net in case of accidental * deletion. Set this value to the number of days you'd like to retain * archived data before it's removed permanently. * - * If 'retainData' is set to false, there will never be any archived data - * to remove. */ archiveRetentionTime: 15, diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 414d064c3..09f5fdfea 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -83,7 +83,6 @@ module.exports.create = function (cfg) { const rpc = cfg.rpc; const tasks = cfg.tasks; const store = cfg.store; - const retainData = cfg.retainData; Log = cfg.log; Log.silly('HK_LOADING', 'LOADING HISTORY_KEEPER MODULE'); @@ -350,18 +349,9 @@ module.exports.create = function (cfg) { but for some reason are still present */ const expireChannel = function (ctx, channel) { - if (retainData) { - return void store.archiveChannel(channel, function (err) { - Log.info("ARCHIVAL_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", { - channelId: channel, - status: err? String(err): "SUCCESS", - }); - }); - } - - store.removeChannel(channel, function (err) { - Log.info("DELETION_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", { - channelid: channel, + return void store.archiveChannel(channel, function (err) { + Log.info("ARCHIVAL_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", { + channelId: channel, status: err? String(err): "SUCCESS", }); }); diff --git a/lib/rpc.js b/lib/rpc.js index dfe27302d..0ece7915c 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -878,22 +878,8 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { })); }).nThen(function (w) { // remove the blob - - if (Env.retainData) { - return void Env.blobStore.archive.blob(blobId, w(function (err) { - Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', { - safeKey: safeKey, - blobId: blobId, - status: err? String(err): 'SUCCESS', - }); - if (err) { - w.abort(); - return void cb(err); - } - })); - } - Env.blobStore.remove.blob(blobId, w(function (err) { - Log.info('DELETION_OWNED_FILE_BY_OWNER_RPC', { + return void Env.blobStore.archive.blob(blobId, w(function (err) { + Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', { safeKey: safeKey, blobId: blobId, status: err? String(err): 'SUCCESS', @@ -904,23 +890,9 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { } })); }).nThen(function () { - // remove the proof - if (Env.retainData) { - return void Env.blobStore.archive.proof(safeKey, blobId, function (err) { - Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", { - safeKey: safeKey, - blobId: blobId, - status: err? String(err): 'SUCCESS', - }); - if (err) { - return void cb("E_PROOF_REMOVAL"); - } - cb(); - }); - } - - Env.blobStore.remove.proof(safeKey, blobId, function (err) { - Log.info("DELETION_PROOF_REMOVAL_BY_OWNER_RPC", { + // archive the proof + return void Env.blobStore.archive.proof(safeKey, blobId, function (err) { + Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", { safeKey: safeKey, blobId: blobId, status: err? String(err): 'SUCCESS', @@ -939,21 +911,9 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { if (!isOwner(metadata, unsafeKey)) { return void cb('INSUFFICIENT_PERMISSIONS'); } - // if the admin has configured data retention... - // temporarily archive the file instead of removing it - if (Env.retainData) { - return void Env.msgStore.archiveChannel(channelId, function (e) { - Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', { - unsafeKey: unsafeKey, - channelId: channelId, - status: e? String(e): 'SUCCESS', - }); - cb(e); - }); - } - - return void Env.msgStore.removeChannel(channelId, function (e) { - Log.info('DELETION_CHANNEL_BY_OWNER_RPC', { + // temporarily archive the file + return void Env.msgStore.archiveChannel(channelId, function (e) { + Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', { unsafeKey: unsafeKey, channelId: channelId, status: e? String(e): 'SUCCESS', @@ -1437,7 +1397,6 @@ RPC.create = function (config, cb) { }; var Env = { - retainData: config.retainData || false, defaultStorageLimit: config.defaultStorageLimit, maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024), Sessions: {}, diff --git a/scripts/evict-inactive.js b/scripts/evict-inactive.js index 13028b8ff..f0e801909 100644 --- a/scripts/evict-inactive.js +++ b/scripts/evict-inactive.js @@ -15,8 +15,6 @@ var inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000); // files which were archived before this date can be considered safe to remove var retentionTime = +new Date() - (config.archiveRetentionTime * 24 * 3600 * 1000); -var retainData = Boolean(config.retainData); - var getNewestTime = function (stats) { return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) { return stats[b] > stats[a]? b: a; @@ -176,23 +174,6 @@ nThen(function (w) { if (pins[item.blobId]) { return void next(); } if (item && getNewestTime(item) > retentionTime) { return void next(); } - if (!retainData) { - return void blobs.remove.blob(item.blobId, function (err) { - if (err) { - Log.error("EVICT_BLOB_ERROR", { - error: err, - item: item, - }); - return void next(); - } - Log.info("EVICT_BLOB_INACTIVE", { - item: item, - }); - removed++; - next(); - }); - } - blobs.archive.blob(item.blobId, function (err) { if (err) { Log.error("EVICT_ARCHIVE_BLOB_ERROR", { @@ -247,7 +228,6 @@ nThen(function (w) { Log.info("EVICT_BLOB_PROOFS_REMOVED", removed); })); }).nThen(function (w) { - var removed = 0; var channels = 0; var archived = 0; @@ -279,42 +259,22 @@ nThen(function (w) { // ignore the channel if it's pinned if (pins[item.channel]) { return void cb(); } - // if the server is configured to retain data, archive the channel - if (config.retainData) { - return void store.archiveChannel(item.channel, w(function (err) { - if (err) { - Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', { - error: err, - channel: item.channel, - }); - return void cb(); - } - Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel); - archived++; - cb(); - })); - } - - // otherwise remove it - store.removeChannel(item.channel, w(function (err) { + return void store.archiveChannel(item.channel, w(function (err) { if (err) { - Log.error('EVICT_CHANNEL_REMOVAL_ERROR', { + Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', { error: err, channel: item.channel, }); return void cb(); } - Log.info('EVICT_CHANNEL_REMOVAL', item.channel); - removed++; + Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel); + archived++; cb(); })); }; var done = function () { - if (config.retainData) { - return void Log.info('EVICT_CHANNELS_ARCHIVED', archived); - } - return void Log.info('EVICT_CHANNELS_REMOVED', removed); + return void Log.info('EVICT_CHANNELS_ARCHIVED', archived); }; store.listChannels(handler, w(done)); diff --git a/server.js b/server.js index fec59a7ea..d2bdcabd8 100644 --- a/server.js +++ b/server.js @@ -278,7 +278,6 @@ var nt = nThen(function (w) { rpc: rpc, store: config.store, log: log, - retainData: Boolean(config.retainData), }; historyKeeper = HK.create(hkConfig); }).nThen(function () { diff --git a/storage/file.js b/storage/file.js index b2899619f..a3d6f521f 100644 --- a/storage/file.js +++ b/storage/file.js @@ -555,9 +555,6 @@ var listChannels = function (root, handler, cb) { // to an equivalent location in the cold storage directory var archiveChannel = function (env, channelName, cb) { // TODO close channels before archiving them? - if (!env.retainData) { - return void cb("ARCHIVES_DISABLED"); - } // ctime is the most reliable indicator of when a file was archived // because it is used to indicate changes to the files metadata @@ -875,7 +872,6 @@ module.exports.create = function (conf, cb) { var env = { root: conf.filePath || './datastore', archiveRoot: conf.archivePath || './data/archive', - retainData: conf.retainData, channels: { }, channelExpirationMs: conf.channelExpirationMs || 30000, verbose: conf.verbose, diff --git a/storage/tasks.js b/storage/tasks.js index 2209b3d59..bb4dbdb9c 100644 --- a/storage/tasks.js +++ b/storage/tasks.js @@ -202,22 +202,6 @@ var expire = function (env, task, cb) { var Log = env.log; var args = task.slice(2); - if (!env.retainData) { - Log.info('DELETION_SCHEDULED_EXPIRATION', { - task: task, - }); - env.store.removeChannel(args[0], function (err) { - if (err) { - Log.error('DELETION_SCHEDULED_EXPIRATION_ERROR', { - task: task, - error: err, - }); - } - cb(); - }); - return; - } - Log.info('ARCHIVAL_SCHEDULED_EXPIRATION', { task: task, }); @@ -381,7 +365,6 @@ Tasks.create = function (config, cb) { root: config.taskPath || './tasks', log: config.log, store: config.store, - retainData: Boolean(config.retainData), }; // make sure the path exists... From 9cdf54aff2def861b3f23a05fa97a91aa7c32149 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 17:58:24 -0500 Subject: [PATCH 18/53] untested implementation of trimHistory --- lib/historyKeeper.js | 30 ++----- lib/hk-util.js | 24 ++++++ storage/file.js | 198 +++++++++++++++++++++++++++++++++++++++---- 3 files changed, 211 insertions(+), 41 deletions(-) create mode 100644 lib/hk-util.js diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 09f5fdfea..c3ed67cb7 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -10,29 +10,13 @@ const Meta = require("./metadata"); const WriteQueue = require("./write-queue"); const BatchRead = require("./batch-read"); +const Extras = require("./hk-util.js"); + let Log; const now = function () { return (new Date()).getTime(); }; const ONE_DAY = 1000 * 60 * 60 * 24; // one day in milliseconds -/* getHash - * this function slices off the leading portion of a message which is - most likely unique - * these "hashes" are used to identify particular messages in a channel's history - * clients store "hashes" either in memory or in their drive to query for new messages: - * when reconnecting to a pad - * when connecting to chat or a mailbox - * thus, we can't change this function without invalidating client data which: - * is encrypted clientside - * can't be easily migrated - * don't break it! -*/ -const getHash = function (msg) { - if (typeof(msg) !== 'string') { - Log.warn('HK_GET_HASH', 'getHash() called on ' + typeof(msg) + ': ' + msg); - return ''; - } - return msg.slice(0,64); -}; +const getHash = Extras.getHash; const tryParse = function (str) { try { @@ -185,7 +169,7 @@ module.exports.create = function (cfg) { if (msg[0] === 0 && msg[2] === 'MSG' && typeof(msg[4]) === 'string') { // msgObj.offset is API guaranteed by our storage module // it should always be a valid positive integer - offsetByHash[getHash(msg[4])] = msgObj.offset; + offsetByHash[getHash(msg[4], Log)] = msgObj.offset; } // There is a trailing \n at the end of the file size = msgObj.offset + msgObj.buff.length + 1; @@ -502,7 +486,7 @@ module.exports.create = function (cfg) { msgStruct.push(now()); // storeMessage - storeMessage(ctx, channel, JSON.stringify(msgStruct), isCp, getHash(msgStruct[4])); + storeMessage(ctx, channel, JSON.stringify(msgStruct), isCp, getHash(msgStruct[4], Log)); }); }; @@ -601,7 +585,7 @@ module.exports.create = function (cfg) { const msg = tryParse(msgObj.buff.toString('utf8')); // if it was undefined then go onto the next message if (typeof msg === "undefined") { return readMore(); } - if (typeof(msg[4]) !== 'string' || lastKnownHash !== getHash(msg[4])) { + if (typeof(msg[4]) !== 'string' || lastKnownHash !== getHash(msg[4], Log)) { return void readMore(); } offset = msgObj.offset; @@ -672,7 +656,7 @@ module.exports.create = function (cfg) { var content = parsed[4]; if (typeof(content) !== 'string') { return; } - var hash = getHash(content); + var hash = getHash(content, Log); if (hash === oldestKnownHash) { found = true; } diff --git a/lib/hk-util.js b/lib/hk-util.js new file mode 100644 index 000000000..cea39c4e1 --- /dev/null +++ b/lib/hk-util.js @@ -0,0 +1,24 @@ +var HK = module.exports; + +/* getHash + * this function slices off the leading portion of a message which is + most likely unique + * these "hashes" are used to identify particular messages in a channel's history + * clients store "hashes" either in memory or in their drive to query for new messages: + * when reconnecting to a pad + * when connecting to chat or a mailbox + * thus, we can't change this function without invalidating client data which: + * is encrypted clientside + * can't be easily migrated + * don't break it! +*/ +HK.getHash = function (msg, Log) { + if (typeof(msg) !== 'string') { + if (Log) { + Log.warn('HK_GET_HASH', 'getHash() called on ' + typeof(msg) + ': ' + msg); + } + return ''; + } + return msg.slice(0,64); +}; + diff --git a/storage/file.js b/storage/file.js index a3d6f521f..08ec98f85 100644 --- a/storage/file.js +++ b/storage/file.js @@ -7,6 +7,8 @@ var Path = require("path"); var nThen = require("nthen"); var Semaphore = require("saferphore"); var Util = require("../lib/common-util"); +var Meta = require("../lib/metadata"); +var Extras = require("../lib/hk-util"); const Schedule = require("../lib/schedule"); const Readline = require("readline"); @@ -39,6 +41,10 @@ var mkArchiveMetadataPath = function (env, channelId) { return Path.join(env.archiveRoot, 'datastore', channelId.slice(0, 2), channelId) + '.metadata.ndjson'; }; +var mkTempPath = function (env, channelId) { + return mkPath(env, channelId) + '.temp'; +}; + // pass in the path so we can reuse the same function for archived files var channelExists = function (filepath, cb) { Fs.stat(filepath, function (err, stat) { @@ -868,6 +874,178 @@ var getMessages = function (env, chanName, handler, cb) { }); }; +var trimChannel = function (env, channelName, hash, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + // this function is queued as a blocking action for the relevant channel + + // derive temporary file paths for metadata and log buffers + var tempChannelPath = mkTempPath(env, channelName); + + // derive production db paths + var channelPath = mkPath(env, channelName); + var metadataPath = mkMetadataPath(env, channelName); + + // derive archive paths + var archiveChannelPath = mkArchivePath(env, channelName); + var archiveMetadataPath = mkArchiveMetadataPath(env, channelName); + + var metadataReference = {}; + + var tempStream; + var ABORT; + + var cleanUp = function (cb) { + if (tempStream && !tempStream.closed) { + try { + tempStream.close(); + } catch (err) { } + } + + Fse.unlink(tempChannelPath, function (err) { + // proceed if deleted or if there was nothing to delete + if (!err || err.code === 'ENOENT') { return cb(); } + // else abort and call back with the error + cb(err); + }); + }; + + nThen(function (w) { + // close the file descriptor if it is open + closeChannel(env, channelName, w(function (err) { + if (err) { + w.abort(); + return void cb(err); + } + })); + }).nThen(function (w) { + cleanUp(w(function (err) { + if (err) { + w.abort(); + cb(err); + } + })); + }).nThen(function (w) { + // eat errors since loading the logger here would create a cyclical dependency + var lineHandler = Meta.createLineHandler(metadataReference, Util.noop); + + readMetadata(env, channelName, lineHandler, w(function (err) { + if (err) { + w.abort(); + return void cb(err); + } + // if there were no errors just fall through to the next block + })); + }).nThen(function (w) { + // create temp buffer writeStream + tempStream = Fs.createWriteStream(tempChannelPath, { + flags: 'a', + }); + tempStream.on('open', w()); + tempStream.on('error', function (err) { + w.abort(); + ABORT = true; + cleanUp(function () { + cb(err); + }); + }); + }).nThen(function (w) { + var i = 0; + var retain = false; + + var handler = function (msgObj, readMore, abort) { + if (ABORT) { return void abort(); } + // the first message might be metadata... ignore it if so + if (i++ === 0 && msgObj.buff.indexOf('{') === 0) { return; } + + if (retain) { + // if this flag is set then you've already found + // the message you were looking for. + // write it to your temp buffer and keep going + return void tempStream.write(msgObj.buff, function () { + readMore(); + }); + } + + var msg = Util.tryParse(msgObj.buff.toString('utf8')); + + var msgHash = Extras.getHash(msg[4]); + + if (msgHash === hash) { + // everything from this point on should be retained + retain = true; + } + }; + + readMessagesBin(env, channelName, 0, handler, w(function (err) { + if (err) { + w.abort(); + return void cleanUp(function () { + // intentionally call back with main error + // not the cleanup error + cb(err); + }); + } + + if (!retain) { + // you never found the message you were looking for + // this whole operation is invalid... + // clean up, abort, and call back with an error + + w.abort(); + cleanUp(function () { + // intentionally call back with main error + // not the cleanup error + cb('HASH_NOT_FOUND'); + }); + } + })); + }).nThen(function (w) { + // copy existing channel to the archive + Fse.copy(channelPath, archiveChannelPath, w(function (err) { + if (!err || err.code === 'ENOENT') { return; } + w.abort(); + cleanUp(function () { + cb(err); + }); + })); + + // copy existing metadaata to the archive + Fse.copy(metadataPath, archiveMetadataPath, w(function (err) { + if (!err || err.code === 'ENOENT') { return; } + w.abort(); + cleanUp(function () { + cb(err); + }); + })); + }).nThen(function (w) { + // overwrite the existing metadata log with the current metadata state + Fs.writeFile(metadataPath, JSON.stringify(metadataReference.meta) + '\n', w(function (err) { + // this shouldn't happen, but if it does your channel might be messed up :( + if (err) { + w.abort(); + cb(err); + } + })); + + // overwrite the existing channel with the temp log + Fse.move(tempChannelPath, channelPath, { + overwrite: true, + }, w(function (err) { + // this shouldn't happen, but if it does your channel might be messed up :( + if (err) { + w.abort(); + cb(err); + } + })); + }).nThen(function () { + // clean up and call back with no error + // triggering a historyKeeper index cache eviction... + cleanUp(function () { + cb(); + }); + }); +}; + module.exports.create = function (conf, cb) { var env = { root: conf.filePath || './datastore', @@ -987,25 +1165,9 @@ module.exports.create = function (conf, cb) { }); }, trimChannel: function (channelName, hash, cb) { - // XXX ansuz - // XXX queue lock - /* block any reads from the metadata and log files - until this whole process has finished - close the file descriptor if it is open - derive temporary file paths for metadata and log buffers - compute metadata state and write to metadata buffer - scan through log file and begin copying lines to the log buffer - once you recognize the first line by the hash the user provided - archive the file and current metadata once both buffers are copied - move the metadata and log buffers into place - return the lock on reads - call back - - in case of an error, remove the buffer files - */ + if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } schedule.blocking(channelName, function (next) { - cb("E_NOT_IMPLEMENTED"); - next(); + trimChannel(env, channelName, hash, Util.both(cb, next)); }); }, From 39b0785406a18a32c2a1a75c0c64ef036ca9dccd Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 09:57:12 -0500 Subject: [PATCH 19/53] apply custom limits immediately at startup --- lib/rpc.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/rpc.js b/lib/rpc.js index 0ece7915c..867cfd283 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -1754,6 +1754,7 @@ RPC.create = function (config, cb) { if (e) { WARN('limitUpdate', e); } + applyCustomLimits(Env, config); }); }; updateLimitDaily(); From ba6e3f33bd25282d23849bab7853cf3d3d70f515 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 11:25:48 -0500 Subject: [PATCH 20/53] move admin commands into their own module --- lib/commands/admin-rpc.js | 121 ++++++++++++++++++++++++++++++++++++++ lib/rpc.js | 116 ++---------------------------------- 2 files changed, 125 insertions(+), 112 deletions(-) create mode 100644 lib/commands/admin-rpc.js diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js new file mode 100644 index 000000000..9e32fe2a8 --- /dev/null +++ b/lib/commands/admin-rpc.js @@ -0,0 +1,121 @@ +/*jshint esversion: 6 */ +const BatchRead = require("../batch-read"); +const nThen = require("nthen"); +const getFolderSize = require("get-folder-size"); +var Fs = require("fs"); + +var Admin = module.exports; + +var getActiveSessions = function (Env, ctx, cb) { + var total = ctx.users ? Object.keys(ctx.users).length : '?'; + + var ips = []; + Object.keys(ctx.users).forEach(function (u) { + var user = ctx.users[u]; + var socket = user.socket; + var req = socket.upgradeReq; + var conn = req && req.connection; + var ip = (req && req.headers && req.headers['x-forwarded-for']) || (conn && conn.remoteAddress); + if (ip && ips.indexOf(ip) === -1) { + ips.push(ip); + } + }); + + cb (void 0, [total, ips.length]); +}; + +var shutdown = function (Env, ctx, cb) { + return void cb('E_NOT_IMPLEMENTED'); + //clearInterval(Env.sessionExpirationInterval); + // XXX set a flag to prevent incoming database writes + // XXX disconnect all users and reject new connections + // XXX wait until all pending writes are complete + // then process.exit(0); + // and allow system functionality to restart the server +}; + +const batchRegisteredUsers = BatchRead("GET_REGISTERED_USERS"); +var getRegisteredUsers = function (Env, cb) { + batchRegisteredUsers('', cb, function (done) { + var dir = Env.paths.pin; + var folders; + var users = 0; + nThen(function (waitFor) { + Fs.readdir(dir, waitFor(function (err, list) { + if (err) { + waitFor.abort(); + return void done(err); + } + folders = list; + })); + }).nThen(function (waitFor) { + folders.forEach(function (f) { + var dir = Env.paths.pin + '/' + f; + Fs.readdir(dir, waitFor(function (err, list) { + if (err) { return; } + users += list.length; + })); + }); + }).nThen(function () { + done(void 0, users); + }); + }); +}; + +const batchDiskUsage = BatchRead("GET_DISK_USAGE"); +var getDiskUsage = function (Env, cb) { + batchDiskUsage('', cb, function (done) { + var data = {}; + nThen(function (waitFor) { + getFolderSize('./', waitFor(function(err, info) { + data.total = info; + })); + getFolderSize(Env.paths.pin, waitFor(function(err, info) { + data.pin = info; + })); + getFolderSize(Env.paths.blob, waitFor(function(err, info) { + data.blob = info; + })); + getFolderSize(Env.paths.staging, waitFor(function(err, info) { + data.blobstage = info; + })); + getFolderSize(Env.paths.block, waitFor(function(err, info) { + data.block = info; + })); + getFolderSize(Env.paths.data, waitFor(function(err, info) { + data.datastore = info; + })); + }).nThen(function () { + done(void 0, data); + }); + }); +}; + + + +Admin.command = function (Env, ctx, publicKey, config, data, cb) { + var admins = Env.admins; + if (admins.indexOf(publicKey) === -1) { + return void cb("FORBIDDEN"); + } + // Handle commands here + switch (data[0]) { + case 'ACTIVE_SESSIONS': + return getActiveSessions(Env, ctx, cb); + case 'ACTIVE_PADS': + return cb(void 0, ctx.channels ? Object.keys(ctx.channels).length : '?'); + case 'REGISTERED_USERS': + return getRegisteredUsers(Env, cb); + case 'DISK_USAGE': + return getDiskUsage(Env, cb); + case 'FLUSH_CACHE': + config.flushCache(); + return cb(void 0, true); + case 'SHUTDOWN': + return shutdown(Env, ctx, cb); + default: + return cb('UNHANDLED_ADMIN_COMMAND'); + } +}; + + diff --git a/lib/rpc.js b/lib/rpc.js index 867cfd283..f96e8a717 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -15,7 +15,6 @@ const Package = require('../package.json'); const Pinned = require('../scripts/pinned'); const Saferphore = require("saferphore"); const nThen = require("nthen"); -const getFolderSize = require("get-folder-size"); const Pins = require("./pins"); const Meta = require("./metadata"); const WriteQueue = require("./write-queue"); @@ -26,6 +25,8 @@ const escapeKeyCharacters = Util.escapeKeyCharacters; const unescapeKeyCharacters = Util.unescapeKeyCharacters; const mkEvent = Util.mkEvent; +const Admin = require("./commands/admin-rpc"); + var RPC = module.exports; var Store = require("../storage/file"); @@ -1207,115 +1208,6 @@ var writePrivateMessage = function (Env, args, nfwssCtx, cb) { }); }; -const batchDiskUsage = BatchRead("GET_DISK_USAGE"); -var getDiskUsage = function (Env, cb) { - batchDiskUsage('', cb, function (done) { - var data = {}; - nThen(function (waitFor) { - getFolderSize('./', waitFor(function(err, info) { - data.total = info; - })); - getFolderSize(Env.paths.pin, waitFor(function(err, info) { - data.pin = info; - })); - getFolderSize(Env.paths.blob, waitFor(function(err, info) { - data.blob = info; - })); - getFolderSize(Env.paths.staging, waitFor(function(err, info) { - data.blobstage = info; - })); - getFolderSize(Env.paths.block, waitFor(function(err, info) { - data.block = info; - })); - getFolderSize(Env.paths.data, waitFor(function(err, info) { - data.datastore = info; - })); - }).nThen(function () { - done(void 0, data); - }); - }); -}; - -const batchRegisteredUsers = BatchRead("GET_REGISTERED_USERS"); -var getRegisteredUsers = function (Env, cb) { - batchRegisteredUsers('', cb, function (done) { - var dir = Env.paths.pin; - var folders; - var users = 0; - nThen(function (waitFor) { - Fs.readdir(dir, waitFor(function (err, list) { - if (err) { - waitFor.abort(); - return void done(err); - } - folders = list; - })); - }).nThen(function (waitFor) { - folders.forEach(function (f) { - var dir = Env.paths.pin + '/' + f; - Fs.readdir(dir, waitFor(function (err, list) { - if (err) { return; } - users += list.length; - })); - }); - }).nThen(function () { - done(void 0, users); - }); - }); -}; -var getActiveSessions = function (Env, ctx, cb) { - var total = ctx.users ? Object.keys(ctx.users).length : '?'; - - var ips = []; - Object.keys(ctx.users).forEach(function (u) { - var user = ctx.users[u]; - var socket = user.socket; - var req = socket.upgradeReq; - var conn = req && req.connection; - var ip = (req && req.headers && req.headers['x-forwarded-for']) || (conn && conn.remoteAddress); - if (ip && ips.indexOf(ip) === -1) { - ips.push(ip); - } - }); - - cb (void 0, [total, ips.length]); -}; - -var shutdown = function (Env, ctx, cb) { - return void cb('E_NOT_IMPLEMENTED'); - //clearInterval(Env.sessionExpirationInterval); - // XXX set a flag to prevent incoming database writes - // XXX disconnect all users and reject new connections - // XXX wait until all pending writes are complete - // then process.exit(0); - // and allow system functionality to restart the server -}; - -var adminCommand = function (Env, ctx, publicKey, config, data, cb) { - var admins = Env.admins; - if (admins.indexOf(publicKey) === -1) { - return void cb("FORBIDDEN"); - } - // Handle commands here - switch (data[0]) { - case 'ACTIVE_SESSIONS': - return getActiveSessions(Env, ctx, cb); - case 'ACTIVE_PADS': - return cb(void 0, ctx.channels ? Object.keys(ctx.channels).length : '?'); - case 'REGISTERED_USERS': - return getRegisteredUsers(Env, cb); - case 'DISK_USAGE': - return getDiskUsage(Env, cb); - case 'FLUSH_CACHE': - config.flushCache(); - return cb(void 0, true); - case 'SHUTDOWN': - return shutdown(Env, ctx, cb); - default: - return cb('UNHANDLED_ADMIN_COMMAND'); - } -}; - var isUnauthenticatedCall = function (call) { return [ 'GET_FILE_SIZE', @@ -1717,7 +1609,7 @@ RPC.create = function (config, cb) { Respond(e); }); case 'ADMIN': - return void adminCommand(Env, ctx, safeKey, config, msg[1], function (e, result) { + return void Admin.command(Env, ctx, safeKey, config, msg[1], function (e, result) { if (e) { WARN(e, result); return void Respond(e); @@ -1754,9 +1646,9 @@ RPC.create = function (config, cb) { if (e) { WARN('limitUpdate', e); } - applyCustomLimits(Env, config); }); }; + applyCustomLimits(Env, config); updateLimitDaily(); setInterval(updateLimitDaily, 24*3600*1000); From 6b5118cdc3a2dff5f2637ef8da36f481439bdafe Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 12:41:42 -0500 Subject: [PATCH 21/53] add an npm script to lint only server components --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index fa4353662..057c823d9 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "package": "PACKAGE=1 node server.js", "lint": "jshint --config .jshintrc --exclude-path .jshintignore . && ./node_modules/lesshint/bin/lesshint -c ./.lesshintrc ./customize.dist/src/less2/", "lint:js": "jshint --config .jshintrc --exclude-path .jshintignore .", + "lint:server": "jshint --config .jshintrc lib", "lint:less": "./node_modules/lesshint/bin/lesshint -c ./.lesshintrc ./customize.dist/src/less2/", "flow": "./node_modules/.bin/flow", "test": "node scripts/TestSelenium.js", From 4fd68b672ec340019427aec6d021f49b4054e3cb Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 12:42:31 -0500 Subject: [PATCH 22/53] drop clientside hooks wrappers for authenticated GET_FILE_SIZE --- www/common/pinpad.js | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/www/common/pinpad.js b/www/common/pinpad.js index 48c62f3da..e9ecbb7f3 100644 --- a/www/common/pinpad.js +++ b/www/common/pinpad.js @@ -89,18 +89,6 @@ var factory = function (Util, Rpc) { }); }; - // get the total stored size of a channel's patches (in bytes) - exp.getFileSize = function (file, cb) { - rpc.send('GET_FILE_SIZE', file, function (e, response) { - if (e) { return void cb(e); } - if (response && response.length && typeof(response[0]) === 'number') { - return void cb(void 0, response[0]); - } else { - cb('INVALID_RESPONSE'); - } - }); - }; - // get the combined size of all channels (in bytes) for all the // channels which the server has pinned for your publicKey exp.getFileListSize = function (cb) { From c93b39c094e8e827024c4f655d144aa103d13be8 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 12:43:11 -0500 Subject: [PATCH 23/53] separate more rpc functionality into pinning and core submodules --- lib/commands/core.js | 42 ++++ lib/commands/pin-rpc.js | 399 +++++++++++++++++++++++++++++++++ lib/rpc.js | 479 +++------------------------------------- 3 files changed, 471 insertions(+), 449 deletions(-) create mode 100644 lib/commands/core.js create mode 100644 lib/commands/pin-rpc.js diff --git a/lib/commands/core.js b/lib/commands/core.js new file mode 100644 index 000000000..7e34232cb --- /dev/null +++ b/lib/commands/core.js @@ -0,0 +1,42 @@ +/*jshint esversion: 6 */ +const Core = module.exports; +const Util = require("../common-util"); +const escapeKeyCharacters = Util.escapeKeyCharacters; + +Core.DEFAULT_LIMIT = 50 * 1024 * 1024; +Core.SESSION_EXPIRATION_TIME = 60 * 1000; + +Core.isValidId = function (chan) { + return chan && chan.length && /^[a-zA-Z0-9=+-]*$/.test(chan) && + [32, 48].indexOf(chan.length) > -1; +}; + +var makeToken = Core.makeToken = function () { + return Number(Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)) + .toString(16); +}; + +Core.getSession = function (Sessions, key) { + var safeKey = escapeKeyCharacters(key); + if (Sessions[safeKey]) { + Sessions[safeKey].atime = +new Date(); + return Sessions[safeKey]; + } + var user = Sessions[safeKey] = {}; + user.atime = +new Date(); + user.tokens = [ + makeToken() + ]; + return user; +}; + + + +// getChannelList +// getSession +// getHash +// getMultipleFileSize +// sumChannelSizes +// getFreeSpace +// getLimit + diff --git a/lib/commands/pin-rpc.js b/lib/commands/pin-rpc.js new file mode 100644 index 000000000..3412f329e --- /dev/null +++ b/lib/commands/pin-rpc.js @@ -0,0 +1,399 @@ +/*jshint esversion: 6 */ +const Core = require("./core"); + +const BatchRead = require("../batch-read"); +const Pins = require("../pins"); + +const Pinning = module.exports; +const Nacl = require("tweetnacl/nacl-fast"); +const Util = require("../common-util"); +const nThen = require("nthen"); + +//const escapeKeyCharacters = Util.escapeKeyCharacters; +const unescapeKeyCharacters = Util.unescapeKeyCharacters; + +var sumChannelSizes = function (sizes) { + return Object.keys(sizes).map(function (id) { return sizes[id]; }) + .filter(function (x) { + // only allow positive numbers + return !(typeof(x) !== 'number' || x <= 0); + }) + .reduce(function (a, b) { return a + b; }, 0); +}; + +// XXX it's possible for this to respond before the server has had a chance +// to fetch the limits. Maybe we should respond with an error... +// or wait until we actually know the limits before responding +var getLimit = Pinning.getLimit = function (Env, publicKey, cb) { + var unescapedKey = unescapeKeyCharacters(publicKey); + var limit = Env.limits[unescapedKey]; + var defaultLimit = typeof(Env.defaultStorageLimit) === 'number'? + Env.defaultStorageLimit: Core.DEFAULT_LIMIT; + + var toSend = limit && typeof(limit.limit) === "number"? + [limit.limit, limit.plan, limit.note] : [defaultLimit, '', '']; + + cb(void 0, toSend); +}; + +var addPinned = function ( + Env, + publicKey /*:string*/, + channelList /*Array*/, + cb /*:()=>void*/) +{ + Env.evPinnedPadsReady.reg(() => { + channelList.forEach((c) => { + const x = Env.pinnedPads[c] = Env.pinnedPads[c] || {}; + x[publicKey] = 1; + }); + cb(); + }); +}; +var removePinned = function ( + Env, + publicKey /*:string*/, + channelList /*Array*/, + cb /*:()=>void*/) +{ + Env.evPinnedPadsReady.reg(() => { + channelList.forEach((c) => { + const x = Env.pinnedPads[c]; + if (!x) { return; } + delete x[publicKey]; + }); + cb(); + }); +}; + +var getMultipleFileSize = function (Env, channels, cb) { + if (!Array.isArray(channels)) { return cb('INVALID_PIN_LIST'); } + if (typeof(Env.msgStore.getChannelSize) !== 'function') { + return cb('GET_CHANNEL_SIZE_UNSUPPORTED'); + } + + var i = channels.length; + var counts = {}; + + var done = function () { + i--; + if (i === 0) { return cb(void 0, counts); } + }; + + channels.forEach(function (channel) { + Pinning.getFileSize(Env, channel, function (e, size) { + if (e) { + // most likely error here is that a file no longer exists + // but a user still has it in their drive, and wants to know + // its size. We should find a way to inform them of this in + // the future. For now we can just tell them it has no size. + + //WARN('getFileSize', e); + counts[channel] = 0; + return done(); + } + counts[channel] = size; + done(); + }); + }); +}; + +const batchUserPins = BatchRead("LOAD_USER_PINS"); +var loadUserPins = function (Env, publicKey, cb) { + var session = Core.getSession(Env.Sessions, publicKey); + + if (session.channels) { + return cb(session.channels); + } + + batchUserPins(publicKey, cb, function (done) { + var ref = {}; + var lineHandler = Pins.createLineHandler(ref, function (label, data) { + Env.Log.error(label, { + log: publicKey, + data: data, + }); + }); + + // if channels aren't in memory. load them from disk + Env.pinStore.getMessages(publicKey, lineHandler, function () { + // no more messages + + // only put this into the cache if it completes + session.channels = ref.pins; + done(ref.pins); // FIXME no error handling? + }); + }); +}; + +var truthyKeys = function (O) { + return Object.keys(O).filter(function (k) { + return O[k]; + }); +}; + +var getChannelList = Pinning.getChannelList = function (Env, publicKey, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + loadUserPins(Env, publicKey, function (pins) { + cb(truthyKeys(pins)); + }); +}; + +const batchTotalSize = BatchRead("GET_TOTAL_SIZE"); +Pinning.getTotalSize = function (Env, publicKey, cb) { + var unescapedKey = unescapeKeyCharacters(publicKey); + var limit = Env.limits[unescapedKey]; + + // Get a common key if multiple users share the same quota, otherwise take the public key + var batchKey = (limit && Array.isArray(limit.users)) ? limit.users.join('') : publicKey; + + batchTotalSize(batchKey, cb, function (done) { + var channels = []; + var bytes = 0; + nThen(function (waitFor) { + // Get the channels list for our user account + Pinning.getChannelList(Env, publicKey, waitFor(function (_channels) { + if (!_channels) { + waitFor.abort(); + return done('INVALID_PIN_LIST'); + } + Array.prototype.push.apply(channels, _channels); + })); + // Get the channels list for users sharing our quota + if (limit && Array.isArray(limit.users) && limit.users.length > 1) { + limit.users.forEach(function (key) { + if (key === unescapedKey) { return; } // Don't count ourselves twice + getChannelList(Env, key, waitFor(function (_channels) { + if (!_channels) { return; } // Broken user, don't count their quota + Array.prototype.push.apply(channels, _channels); + })); + }); + } + }).nThen(function (waitFor) { + // Get size of the channels + var list = []; // Contains the channels already counted in the quota to avoid duplicates + channels.forEach(function (channel) { // TODO semaphore? + if (list.indexOf(channel) !== -1) { return; } + list.push(channel); + Pinning.getFileSize(Env, channel, waitFor(function (e, size) { + if (!e) { bytes += size; } + })); + }); + }).nThen(function () { + done(void 0, bytes); + }); + }); +}; + +/* Users should be able to clear their own pin log with an authenticated RPC +*/ +Pinning.removePins = function (Env, safeKey, cb) { + if (typeof(Env.pinStore.removeChannel) !== 'function') { + return void cb("E_NOT_IMPLEMENTED"); + } + Env.pinStore.removeChannel(safeKey, function (err) { + Env.Log.info('DELETION_PIN_BY_OWNER_RPC', { + safeKey: safeKey, + status: err? String(err): 'SUCCESS', + }); + + cb(err); + }); +}; + +Pinning.trimPins = function (Env, safeKey, cb) { + // XXX trim to latest pin checkpoint + cb("NOT_IMPLEMENTED"); +}; + +var getFreeSpace = Pinning.getFreeSpace = function (Env, publicKey, cb) { + getLimit(Env, publicKey, function (e, limit) { + if (e) { return void cb(e); } + Pinning.getTotalSize(Env, publicKey, function (e, size) { + if (typeof(size) === 'undefined') { return void cb(e); } + + var rem = limit[0] - size; + if (typeof(rem) !== 'number') { + return void cb('invalid_response'); + } + cb(void 0, rem); + }); + }); +}; + +var hashChannelList = function (A) { + var uniques = []; + + A.forEach(function (a) { + if (uniques.indexOf(a) === -1) { uniques.push(a); } + }); + uniques.sort(); + + var hash = Nacl.util.encodeBase64(Nacl.hash(Nacl + .util.decodeUTF8(JSON.stringify(uniques)))); + + return hash; +}; + +var getHash = Pinning.getHash = function (Env, publicKey, cb) { + getChannelList(Env, publicKey, function (channels) { + cb(void 0, hashChannelList(channels)); + }); +}; + +Pinning.pinChannel = function (Env, publicKey, channels, cb) { + if (!channels && channels.filter) { + return void cb('INVALID_PIN_LIST'); + } + + // get channel list ensures your session has a cached channel list + getChannelList(Env, publicKey, function (pinned) { + var session = Core.getSession(Env.Sessions, publicKey); + + // only pin channels which are not already pinned + var toStore = channels.filter(function (channel) { + return pinned.indexOf(channel) === -1; + }); + + if (toStore.length === 0) { + return void getHash(Env, publicKey, cb); + } + + getMultipleFileSize(Env, toStore, function (e, sizes) { + if (typeof(sizes) === 'undefined') { return void cb(e); } + var pinSize = sumChannelSizes(sizes); + + getFreeSpace(Env, publicKey, function (e, free) { + if (typeof(free) === 'undefined') { + Env.WARN('getFreeSpace', e); + return void cb(e); + } + if (pinSize > free) { return void cb('E_OVER_LIMIT'); } + + Env.pinStore.message(publicKey, JSON.stringify(['PIN', toStore, +new Date()]), + function (e) { + if (e) { return void cb(e); } + toStore.forEach(function (channel) { + session.channels[channel] = true; + }); + addPinned(Env, publicKey, toStore, () => {}); + getHash(Env, publicKey, cb); + }); + }); + }); + }); +}; + +Pinning.unpinChannel = function (Env, publicKey, channels, cb) { + if (!channels && channels.filter) { + // expected array + return void cb('INVALID_PIN_LIST'); + } + + getChannelList(Env, publicKey, function (pinned) { + var session = Core.getSession(Env.Sessions, publicKey); + + // only unpin channels which are pinned + var toStore = channels.filter(function (channel) { + return pinned.indexOf(channel) !== -1; + }); + + if (toStore.length === 0) { + return void getHash(Env, publicKey, cb); + } + + Env.pinStore.message(publicKey, JSON.stringify(['UNPIN', toStore, +new Date()]), + function (e) { + if (e) { return void cb(e); } + toStore.forEach(function (channel) { + delete session.channels[channel]; + }); + removePinned(Env, publicKey, toStore, () => {}); + getHash(Env, publicKey, cb); + }); + }); +}; + +Pinning.resetUserPins = function (Env, publicKey, channelList, cb) { + if (!Array.isArray(channelList)) { return void cb('INVALID_PIN_LIST'); } + var session = Core.getSession(Env.Sessions, publicKey); + + if (!channelList.length) { + return void getHash(Env, publicKey, function (e, hash) { + if (e) { return cb(e); } + cb(void 0, hash); + }); + } + + var pins = {}; + getMultipleFileSize(Env, channelList, function (e, sizes) { + if (typeof(sizes) === 'undefined') { return void cb(e); } + var pinSize = sumChannelSizes(sizes); + + + getLimit(Env, publicKey, function (e, limit) { + if (e) { + Env.WARN('[RESET_ERR]', e); + return void cb(e); + } + + /* we want to let people pin, even if they are over their limit, + but they should only be able to do this once. + + This prevents data loss in the case that someone registers, but + does not have enough free space to pin their migrated data. + + They will not be able to pin additional pads until they upgrade + or delete enough files to go back under their limit. */ + if (pinSize > limit[0] && session.hasPinned) { return void(cb('E_OVER_LIMIT')); } + Env.pinStore.message(publicKey, JSON.stringify(['RESET', channelList, +new Date()]), + function (e) { + if (e) { return void cb(e); } + channelList.forEach(function (channel) { + pins[channel] = true; + }); + + var oldChannels; + if (session.channels && typeof(session.channels) === 'object') { + oldChannels = Object.keys(session.channels); + } else { + oldChannels = []; + } + removePinned(Env, publicKey, oldChannels, () => { + addPinned(Env, publicKey, channelList, ()=>{}); + }); + + // update in-memory cache IFF the reset was allowed. + session.channels = pins; + getHash(Env, publicKey, function (e, hash) { + cb(e, hash); + }); + }); + }); + }); +}; + +Pinning.getFileSize = function (Env, channel, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } + if (channel.length === 32) { + if (typeof(Env.msgStore.getChannelSize) !== 'function') { + return cb('GET_CHANNEL_SIZE_UNSUPPORTED'); + } + + return void Env.msgStore.getChannelSize(channel, function (e, size /*:number*/) { + if (e) { + if (e.code === 'ENOENT') { return void cb(void 0, 0); } + return void cb(e.code); + } + cb(void 0, size); + }); + } + + // 'channel' refers to a file, so you need another API + Env.blobStore.size(channel, function (e, size) { + if (typeof(size) === 'undefined') { return void cb(e); } + cb(void 0, size); + }); +}; + diff --git a/lib/rpc.js b/lib/rpc.js index f96e8a717..a064ebe64 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -15,10 +15,10 @@ const Package = require('../package.json'); const Pinned = require('../scripts/pinned'); const Saferphore = require("saferphore"); const nThen = require("nthen"); -const Pins = require("./pins"); const Meta = require("./metadata"); const WriteQueue = require("./write-queue"); const BatchRead = require("./batch-read"); +const Core = require("./commands/core"); const Util = require("./common-util"); const escapeKeyCharacters = Util.escapeKeyCharacters; @@ -26,15 +26,13 @@ const unescapeKeyCharacters = Util.unescapeKeyCharacters; const mkEvent = Util.mkEvent; const Admin = require("./commands/admin-rpc"); +const Pinning = require("./commands/pin-rpc"); var RPC = module.exports; var Store = require("../storage/file"); var BlobStore = require("../storage/blob"); -var DEFAULT_LIMIT = 50 * 1024 * 1024; -var SESSION_EXPIRATION_TIME = 60 * 1000; - var Log; var WARN = function (e, output) { @@ -47,16 +45,6 @@ var WARN = function (e, output) { } }; -var isValidId = function (chan) { - return chan && chan.length && /^[a-zA-Z0-9=+-]*$/.test(chan) && - [32, 48].indexOf(chan.length) > -1; -}; - -var makeToken = function () { - return Number(Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)) - .toString(16); -}; - var makeCookie = function (token) { var time = (+new Date()); time -= time % 5000; @@ -81,20 +69,6 @@ var parseCookie = function (cookie) { return c; }; -var getSession = function (Sessions, key) { - var safeKey = escapeKeyCharacters(key); - if (Sessions[safeKey]) { - Sessions[safeKey].atime = +new Date(); - return Sessions[safeKey]; - } - var user = Sessions[safeKey] = {}; - user.atime = +new Date(); - user.tokens = [ - makeToken() - ]; - return user; -}; - var isTooOld = function (time, now) { return (now - time) > 300000; }; @@ -121,7 +95,7 @@ var expireSessions = function (Sessions) { var addTokenForKey = function (Sessions, publicKey, token) { if (!Sessions[publicKey]) { throw new Error('undefined user'); } - var user = getSession(Sessions, publicKey); + var user = Core.getSession(Sessions, publicKey); user.tokens.push(token); user.atime = +new Date(); if (user.tokens.length > 2) { user.tokens.shift(); } @@ -143,7 +117,7 @@ var isValidCookie = function (Sessions, publicKey, cookie) { return false; } - var user = getSession(Sessions, publicKey); + var user = Core.getSession(Sessions, publicKey); if (!user) { return false; } var idx = user.tokens.indexOf(parsed.seq); @@ -151,7 +125,7 @@ var isValidCookie = function (Sessions, publicKey, cookie) { if (idx > 0) { // make a new token - addTokenForKey(Sessions, publicKey, makeToken()); + addTokenForKey(Sessions, publicKey, Core.makeToken()); } return true; @@ -195,74 +169,9 @@ var checkSignature = function (signedMsg, signature, publicKey) { return Nacl.sign.detached.verify(signedBuffer, signatureBuffer, pubBuffer); }; -const batchUserPins = BatchRead("LOAD_USER_PINS"); -var loadUserPins = function (Env, publicKey, cb) { - var session = getSession(Env.Sessions, publicKey); - - if (session.channels) { - return cb(session.channels); - } - - batchUserPins(publicKey, cb, function (done) { - var ref = {}; - var lineHandler = Pins.createLineHandler(ref, function (label, data) { - Log.error(label, { - log: publicKey, - data: data, - }); - }); - - // if channels aren't in memory. load them from disk - Env.pinStore.getMessages(publicKey, lineHandler, function () { - // no more messages - - // only put this into the cache if it completes - session.channels = ref.pins; - done(ref.pins); // FIXME no error handling? - }); - }); -}; - -var truthyKeys = function (O) { - return Object.keys(O).filter(function (k) { - return O[k]; - }); -}; - -var getChannelList = function (Env, publicKey, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - loadUserPins(Env, publicKey, function (pins) { - cb(truthyKeys(pins)); - }); -}; - -var getFileSize = function (Env, channel, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - if (!isValidId(channel)) { return void cb('INVALID_CHAN'); } - if (channel.length === 32) { - if (typeof(Env.msgStore.getChannelSize) !== 'function') { - return cb('GET_CHANNEL_SIZE_UNSUPPORTED'); - } - - return void Env.msgStore.getChannelSize(channel, function (e, size /*:number*/) { - if (e) { - if (e.code === 'ENOENT') { return void cb(void 0, 0); } - return void cb(e.code); - } - cb(void 0, size); - }); - } - - // 'channel' refers to a file, so you need another API - Env.blobStore.size(channel, function (e, size) { - if (typeof(size) === 'undefined') { return void cb(e); } - cb(void 0, size); - }); -}; - const batchMetadata = BatchRead("GET_METADATA"); var getMetadata = function (Env, channel, cb) { - if (!isValidId(channel)) { return void cb('INVALID_CHAN'); } + if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } if (channel.length !== 32) { return cb("INVALID_CHAN_LENGTH"); } batchMetadata(channel, cb, function (done) { @@ -309,7 +218,7 @@ var queueMetadata = WriteQueue(); var setMetadata = function (Env, data, unsafeKey, cb) { var channel = data.channel; var command = data.command; - if (!channel || !isValidId(channel)) { return void cb ('INVALID_CHAN'); } + if (!channel || !Core.isValidId(channel)) { return void cb ('INVALID_CHAN'); } if (!command || typeof (command) !== 'string') { return void cb ('INVALID_COMMAND'); } if (Meta.commands.indexOf(command) === -1) { return void('UNSUPPORTED_COMMAND'); } @@ -382,38 +291,6 @@ var setMetadata = function (Env, data, unsafeKey, cb) { }); }; -var getMultipleFileSize = function (Env, channels, cb) { - if (!Array.isArray(channels)) { return cb('INVALID_PIN_LIST'); } - if (typeof(Env.msgStore.getChannelSize) !== 'function') { - return cb('GET_CHANNEL_SIZE_UNSUPPORTED'); - } - - var i = channels.length; - var counts = {}; - - var done = function () { - i--; - if (i === 0) { return cb(void 0, counts); } - }; - - channels.forEach(function (channel) { - getFileSize(Env, channel, function (e, size) { - if (e) { - // most likely error here is that a file no longer exists - // but a user still has it in their drive, and wants to know - // its size. We should find a way to inform them of this in - // the future. For now we can just tell them it has no size. - - //WARN('getFileSize', e); - counts[channel] = 0; - return done(); - } - counts[channel] = size; - done(); - }); - }); -}; - /* accepts a list, and returns a sublist of channel or file ids which seem to have been deleted from the server (file size 0) @@ -429,7 +306,7 @@ var getDeletedPads = function (Env, channels, cb) { var job = function (channel, wait) { return function (give) { - getFileSize(Env, channel, wait(give(function (e, size) { + Pinning.getFileSize(Env, channel, wait(give(function (e, size) { if (e) { return; } if (size === 0) { absentees.push(channel); } }))); @@ -445,72 +322,6 @@ var getDeletedPads = function (Env, channels, cb) { }); }; -const batchTotalSize = BatchRead("GET_TOTAL_SIZE"); -var getTotalSize = function (Env, publicKey, cb) { - var unescapedKey = unescapeKeyCharacters(publicKey); - var limit = Env.limits[unescapedKey]; - - // Get a common key if multiple users share the same quota, otherwise take the public key - var batchKey = (limit && Array.isArray(limit.users)) ? limit.users.join('') : publicKey; - - batchTotalSize(batchKey, cb, function (done) { - var channels = []; - var bytes = 0; - nThen(function (waitFor) { - // Get the channels list for our user account - getChannelList(Env, publicKey, waitFor(function (_channels) { - if (!_channels) { - waitFor.abort(); - return done('INVALID_PIN_LIST'); - } - Array.prototype.push.apply(channels, _channels); - })); - // Get the channels list for users sharing our quota - if (limit && Array.isArray(limit.users) && limit.users.length > 1) { - limit.users.forEach(function (key) { - if (key === unescapedKey) { return; } // Don't count ourselves twice - getChannelList(Env, key, waitFor(function (_channels) { - if (!_channels) { return; } // Broken user, don't count their quota - Array.prototype.push.apply(channels, _channels); - })); - }); - } - }).nThen(function (waitFor) { - // Get size of the channels - var list = []; // Contains the channels already counted in the quota to avoid duplicates - channels.forEach(function (channel) { // TODO semaphore? - if (list.indexOf(channel) !== -1) { return; } - list.push(channel); - getFileSize(Env, channel, waitFor(function (e, size) { - if (!e) { bytes += size; } - })); - }); - }).nThen(function () { - done(void 0, bytes); - }); - }); -}; - -var hashChannelList = function (A) { - var uniques = []; - - A.forEach(function (a) { - if (uniques.indexOf(a) === -1) { uniques.push(a); } - }); - uniques.sort(); - - var hash = Nacl.util.encodeBase64(Nacl.hash(Nacl - .util.decodeUTF8(JSON.stringify(uniques)))); - - return hash; -}; - -var getHash = function (Env, publicKey, cb) { - getChannelList(Env, publicKey, function (channels) { - cb(void 0, hashChannelList(channels)); - }); -}; - var applyCustomLimits = function (Env, config) { var isLimit = function (o) { var valid = o && typeof(o) === 'object' && @@ -551,7 +362,7 @@ var updateLimits = function (Env, config, publicKey, cb /*:(?string, ?any[])=>vo if (typeof cb !== "function") { cb = function () {}; } var defaultLimit = typeof(config.defaultStorageLimit) === 'number'? - config.defaultStorageLimit: DEFAULT_LIMIT; + config.defaultStorageLimit: Core.DEFAULT_LIMIT; var userId; if (publicKey) { @@ -612,45 +423,6 @@ var updateLimits = function (Env, config, publicKey, cb /*:(?string, ?any[])=>vo req.end(body); }; -// XXX it's possible for this to respond before the server has had a chance -// to fetch the limits. Maybe we should respond with an error... -// or wait until we actually know the limits before responding -var getLimit = function (Env, publicKey, cb) { - var unescapedKey = unescapeKeyCharacters(publicKey); - var limit = Env.limits[unescapedKey]; - var defaultLimit = typeof(Env.defaultStorageLimit) === 'number'? - Env.defaultStorageLimit: DEFAULT_LIMIT; - - var toSend = limit && typeof(limit.limit) === "number"? - [limit.limit, limit.plan, limit.note] : [defaultLimit, '', '']; - - cb(void 0, toSend); -}; - -var getFreeSpace = function (Env, publicKey, cb) { - getLimit(Env, publicKey, function (e, limit) { - if (e) { return void cb(e); } - getTotalSize(Env, publicKey, function (e, size) { - if (typeof(size) === 'undefined') { return void cb(e); } - - var rem = limit[0] - size; - if (typeof(rem) !== 'number') { - return void cb('invalid_response'); - } - cb(void 0, rem); - }); - }); -}; - -var sumChannelSizes = function (sizes) { - return Object.keys(sizes).map(function (id) { return sizes[id]; }) - .filter(function (x) { - // only allow positive numbers - return !(typeof(x) !== 'number' || x <= 0); - }) - .reduce(function (a, b) { return a + b; }, 0); -}; - // inform that the var loadChannelPins = function (Env) { Pinned.load(function (err, data) { @@ -670,35 +442,7 @@ var loadChannelPins = function (Env) { pinPath: Env.paths.pin, }); }; -var addPinned = function ( - Env, - publicKey /*:string*/, - channelList /*Array*/, - cb /*:()=>void*/) -{ - Env.evPinnedPadsReady.reg(() => { - channelList.forEach((c) => { - const x = Env.pinnedPads[c] = Env.pinnedPads[c] || {}; - x[publicKey] = 1; - }); - cb(); - }); -}; -var removePinned = function ( - Env, - publicKey /*:string*/, - channelList /*Array*/, - cb /*:()=>void*/) -{ - Env.evPinnedPadsReady.reg(() => { - channelList.forEach((c) => { - const x = Env.pinnedPads[c]; - if (!x) { return; } - delete x[publicKey]; - }); - cb(); - }); -}; + var isChannelPinned = function (Env, channel, cb) { Env.evPinnedPadsReady.reg(() => { if (Env.pinnedPads[channel] && Object.keys(Env.pinnedPads[channel]).length) { @@ -710,138 +454,6 @@ var isChannelPinned = function (Env, channel, cb) { }); }; -var pinChannel = function (Env, publicKey, channels, cb) { - if (!channels && channels.filter) { - return void cb('INVALID_PIN_LIST'); - } - - // get channel list ensures your session has a cached channel list - getChannelList(Env, publicKey, function (pinned) { - var session = getSession(Env.Sessions, publicKey); - - // only pin channels which are not already pinned - var toStore = channels.filter(function (channel) { - return pinned.indexOf(channel) === -1; - }); - - if (toStore.length === 0) { - return void getHash(Env, publicKey, cb); - } - - getMultipleFileSize(Env, toStore, function (e, sizes) { - if (typeof(sizes) === 'undefined') { return void cb(e); } - var pinSize = sumChannelSizes(sizes); - - getFreeSpace(Env, publicKey, function (e, free) { - if (typeof(free) === 'undefined') { - WARN('getFreeSpace', e); - return void cb(e); - } - if (pinSize > free) { return void cb('E_OVER_LIMIT'); } - - Env.pinStore.message(publicKey, JSON.stringify(['PIN', toStore, +new Date()]), - function (e) { - if (e) { return void cb(e); } - toStore.forEach(function (channel) { - session.channels[channel] = true; - }); - addPinned(Env, publicKey, toStore, () => {}); - getHash(Env, publicKey, cb); - }); - }); - }); - }); -}; - -var unpinChannel = function (Env, publicKey, channels, cb) { - if (!channels && channels.filter) { - // expected array - return void cb('INVALID_PIN_LIST'); - } - - getChannelList(Env, publicKey, function (pinned) { - var session = getSession(Env.Sessions, publicKey); - - // only unpin channels which are pinned - var toStore = channels.filter(function (channel) { - return pinned.indexOf(channel) !== -1; - }); - - if (toStore.length === 0) { - return void getHash(Env, publicKey, cb); - } - - Env.pinStore.message(publicKey, JSON.stringify(['UNPIN', toStore, +new Date()]), - function (e) { - if (e) { return void cb(e); } - toStore.forEach(function (channel) { - delete session.channels[channel]; - }); - removePinned(Env, publicKey, toStore, () => {}); - getHash(Env, publicKey, cb); - }); - }); -}; - -var resetUserPins = function (Env, publicKey, channelList, cb) { - if (!Array.isArray(channelList)) { return void cb('INVALID_PIN_LIST'); } - var session = getSession(Env.Sessions, publicKey); - - if (!channelList.length) { - return void getHash(Env, publicKey, function (e, hash) { - if (e) { return cb(e); } - cb(void 0, hash); - }); - } - - var pins = {}; - getMultipleFileSize(Env, channelList, function (e, sizes) { - if (typeof(sizes) === 'undefined') { return void cb(e); } - var pinSize = sumChannelSizes(sizes); - - - getLimit(Env, publicKey, function (e, limit) { - if (e) { - WARN('[RESET_ERR]', e); - return void cb(e); - } - - /* we want to let people pin, even if they are over their limit, - but they should only be able to do this once. - - This prevents data loss in the case that someone registers, but - does not have enough free space to pin their migrated data. - - They will not be able to pin additional pads until they upgrade - or delete enough files to go back under their limit. */ - if (pinSize > limit[0] && session.hasPinned) { return void(cb('E_OVER_LIMIT')); } - Env.pinStore.message(publicKey, JSON.stringify(['RESET', channelList, +new Date()]), - function (e) { - if (e) { return void cb(e); } - channelList.forEach(function (channel) { - pins[channel] = true; - }); - - var oldChannels; - if (session.channels && typeof(session.channels) === 'object') { - oldChannels = Object.keys(session.channels); - } else { - oldChannels = []; - } - removePinned(Env, publicKey, oldChannels, () => { - addPinned(Env, publicKey, channelList, ()=>{}); - }); - - // update in-memory cache IFF the reset was allowed. - session.channels = pins; - getHash(Env, publicKey, function (e, hash) { - cb(e, hash); - }); - }); - }); - }); -}; - var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { if (typeof(channelId) !== 'string' || channelId.length !== 32) { return cb('INVALID_ARGUMENTS'); @@ -861,7 +473,7 @@ var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { }; var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { - if (typeof(channelId) !== 'string' || !isValidId(channelId)) { + if (typeof(channelId) !== 'string' || !Core.isValidId(channelId)) { return cb('INVALID_ARGUMENTS'); } @@ -948,26 +560,6 @@ var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { }); }; -/* Users should be able to clear their own pin log with an authenticated RPC -*/ -var removePins = function (Env, safeKey, cb) { - if (typeof(Env.pinStore.removeChannel) !== 'function') { - return void cb("E_NOT_IMPLEMENTED"); - } - Env.pinStore.removeChannel(safeKey, function (err) { - Log.info('DELETION_PIN_BY_OWNER_RPC', { - safeKey: safeKey, - status: err? String(err): 'SUCCESS', - }); - - cb(err); - }); -}; - -var trimPins = function (Env, safeKey, cb) { - // XXX trim to latest pin checkpoint - cb("NOT_IMPLEMENTED"); -}; /* We assume that the server is secured against MitM attacks @@ -1136,7 +728,7 @@ var ARRAY_LINE = /^\[/; otherwise false */ var isNewChannel = function (Env, channel, cb) { - if (!isValidId(channel)) { return void cb('INVALID_CHAN'); } + if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } if (channel.length !== 32) { return void cb('INVALID_CHAN'); } var done = false; @@ -1172,7 +764,7 @@ var writePrivateMessage = function (Env, args, nfwssCtx, cb) { if (!msg) { return void cb("INVALID_MESSAGE"); } // don't support anything except regular channels - if (!isValidId(channelId) || channelId.length !== 32) { + if (!Core.isValidId(channelId) || channelId.length !== 32) { return void cb("INVALID_CHAN"); } @@ -1270,7 +862,7 @@ var upload_status = function (Env, safeKey, filesize, _cb) { // FIXME FILES }).nThen(function () { // if yuo're here then there are no pending uploads // check if you have space in your quota to upload something of this size - getFreeSpace(Env, safeKey, function (e, free) { + Pinning.getFreeSpace(Env, safeKey, function (e, free) { if (e) { return void cb(e); } if (filesize >= free) { return cb('NOT_ENOUGH_SPACE'); } cb(void 0, false); @@ -1300,6 +892,8 @@ RPC.create = function (config, cb) { limits: {}, admins: [], sessionExpirationInterval: undefined, + Log: Log, + WARN: WARN, }; try { @@ -1344,7 +938,7 @@ RPC.create = function (config, cb) { break; } case 'GET_FILE_SIZE': - return void getFileSize(Env, msg[1], function (e, size) { + return void Pinning.getFileSize(Env, msg[1], function (e, size) { WARN(e, msg[1]); respond(e, [null, size, null]); }); @@ -1354,7 +948,7 @@ RPC.create = function (config, cb) { respond(e, [null, data, null]); }); case 'GET_MULTIPLE_FILE_SIZE': - return void getMultipleFileSize(Env, msg[1], function (e, dict) { + return void Pinning.getMultipleFileSize(Env, msg[1], function (e, dict) { if (e) { WARN(e, dict); return respond(e); @@ -1414,7 +1008,7 @@ RPC.create = function (config, cb) { // make sure a user object is initialized in the cookie jar if (publicKey) { - getSession(Sessions, publicKey); + Core.getSession(Sessions, publicKey); } else { Log.debug("NO_PUBLIC_KEY_PROVIDED", publicKey); } @@ -1471,38 +1065,33 @@ RPC.create = function (config, cb) { switch (msg[0]) { case 'COOKIE': return void Respond(void 0); case 'RESET': - return resetUserPins(Env, safeKey, msg[1], function (e, hash) { + return Pinning.resetUserPins(Env, safeKey, msg[1], function (e, hash) { //WARN(e, hash); return void Respond(e, hash); }); case 'PIN': - return pinChannel(Env, safeKey, msg[1], function (e, hash) { + return Pinning.pinChannel(Env, safeKey, msg[1], function (e, hash) { WARN(e, hash); Respond(e, hash); }); case 'UNPIN': - return unpinChannel(Env, safeKey, msg[1], function (e, hash) { + return Pinning.unpinChannel(Env, safeKey, msg[1], function (e, hash) { WARN(e, hash); Respond(e, hash); }); case 'GET_HASH': - return void getHash(Env, safeKey, function (e, hash) { + return void Pinning.getHash(Env, safeKey, function (e, hash) { WARN(e, hash); Respond(e, hash); }); case 'GET_TOTAL_SIZE': // TODO cache this, since it will get called quite a bit - return getTotalSize(Env, safeKey, function (e, size) { + return Pinning.getTotalSize(Env, safeKey, function (e, size) { if (e) { WARN(e, safeKey); return void Respond(e); } Respond(e, size); }); - case 'GET_FILE_SIZE': - return void getFileSize(Env, msg[1], function (e, size) { - WARN(e, msg[1]); - Respond(e, size); - }); case 'UPDATE_LIMITS': return void updateLimits(Env, config, safeKey, function (e, limit) { if (e) { @@ -1512,21 +1101,13 @@ RPC.create = function (config, cb) { Respond(void 0, limit); }); case 'GET_LIMIT': - return void getLimit(Env, safeKey, function (e, limit) { + return void Pinning.getLimit(Env, safeKey, function (e, limit) { if (e) { WARN(e, limit); return void Respond(e); } Respond(void 0, limit); }); - case 'GET_MULTIPLE_FILE_SIZE': - return void getMultipleFileSize(Env, msg[1], function (e, dict) { - if (e) { - WARN(e, dict); - return void Respond(e); - } - Respond(void 0, dict); - }); case 'EXPIRE_SESSION': return void setTimeout(function () { expireSession(Sessions, safeKey); @@ -1549,12 +1130,12 @@ RPC.create = function (config, cb) { Respond(void 0, 'OK'); }); case 'REMOVE_PINS': - return void removePins(Env, safeKey, function (e) { + return void Pinning.removePins(Env, safeKey, function (e) { if (e) { return void Respond(e); } Respond(void 0, "OK"); }); case 'TRIM_PINS': - return void trimPins(Env, safeKey, function (e) { + return void Pinning.trimPins(Env, safeKey, function (e) { if (e) { return void Respond(e); } Respond(void 0, "OK"); }); @@ -1568,7 +1149,7 @@ RPC.create = function (config, cb) { return void upload_status(Env, safeKey, filesize, function (e, yes) { if (!e && !yes) { // no pending uploads, set the new size - var user = getSession(Sessions, safeKey); + var user = Core.getSession(Sessions, safeKey); user.pendingUploadSize = filesize; user.currentUploadSize = 0; } @@ -1665,7 +1246,7 @@ RPC.create = function (config, cb) { blobStagingPath: config.blobStagingPath, archivePath: config.archivePath, getSession: function (safeKey) { - return getSession(Sessions, safeKey); + return Core.getSession(Sessions, safeKey); }, }, w(function (err, blob) { if (err) { throw new Error(err); } @@ -1677,6 +1258,6 @@ RPC.create = function (config, cb) { // XXX allow for graceful shutdown Env.sessionExpirationInterval = setInterval(function () { expireSessions(Sessions); - }, SESSION_EXPIRATION_TIME); + }, Core.SESSION_EXPIRATION_TIME); }); }; From c765362744b4cfb143fc29ac684b5d79be4c6436 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 13:06:46 -0500 Subject: [PATCH 24/53] move more rpc functionality into modules --- lib/commands/core.js | 153 ++++++++++++++++- lib/commands/pin-rpc.js | 65 ++++++++ lib/commands/quota.js | 111 +++++++++++++ lib/rpc.js | 359 +++------------------------------------- 4 files changed, 346 insertions(+), 342 deletions(-) create mode 100644 lib/commands/quota.js diff --git a/lib/commands/core.js b/lib/commands/core.js index 7e34232cb..0bf7ff542 100644 --- a/lib/commands/core.js +++ b/lib/commands/core.js @@ -1,8 +1,13 @@ /*jshint esversion: 6 */ +/* globals process */ const Core = module.exports; const Util = require("../common-util"); const escapeKeyCharacters = Util.escapeKeyCharacters; +/* Use Nacl for checking signatures of messages */ +const Nacl = require("tweetnacl/nacl-fast"); + + Core.DEFAULT_LIMIT = 50 * 1024 * 1024; Core.SESSION_EXPIRATION_TIME = 60 * 1000; @@ -16,6 +21,30 @@ var makeToken = Core.makeToken = function () { .toString(16); }; +Core.makeCookie = function (token) { + var time = (+new Date()); + time -= time % 5000; + + return [ + time, + process.pid, + token + ]; +}; + +var parseCookie = function (cookie) { + if (!(cookie && cookie.split)) { return null; } + + var parts = cookie.split('|'); + if (parts.length !== 3) { return null; } + + var c = {}; + c.time = new Date(parts[0]); + c.pid = Number(parts[1]); + c.seq = parts[2]; + return c; +}; + Core.getSession = function (Sessions, key) { var safeKey = escapeKeyCharacters(key); if (Sessions[safeKey]) { @@ -30,13 +59,123 @@ Core.getSession = function (Sessions, key) { return user; }; +Core.expireSession = function (Sessions, key) { + var session = Sessions[key]; + if (!session) { return; } + if (session.blobstage) { + session.blobstage.close(); + } + delete Sessions[key]; +}; + +var isTooOld = function (time, now) { + return (now - time) > 300000; +}; + +Core.expireSessions = function (Sessions) { + var now = +new Date(); + Object.keys(Sessions).forEach(function (key) { + var session = Sessions[key]; + if (session && isTooOld(session.atime, now)) { + Core.expireSession(Sessions, key); + } + }); +}; + +var addTokenForKey = function (Sessions, publicKey, token) { + if (!Sessions[publicKey]) { throw new Error('undefined user'); } + + var user = Core.getSession(Sessions, publicKey); + user.tokens.push(token); + user.atime = +new Date(); + if (user.tokens.length > 2) { user.tokens.shift(); } +}; + +Core.isValidCookie = function (Sessions, publicKey, cookie) { + var parsed = parseCookie(cookie); + if (!parsed) { return false; } + + var now = +new Date(); + + if (!parsed.time) { return false; } + if (isTooOld(parsed.time, now)) { + return false; + } + + // different process. try harder + if (process.pid !== parsed.pid) { + return false; + } + + var user = Core.getSession(Sessions, publicKey); + if (!user) { return false; } + + var idx = user.tokens.indexOf(parsed.seq); + if (idx === -1) { return false; } + + if (idx > 0) { + // make a new token + addTokenForKey(Sessions, publicKey, Core.makeToken()); + } + + return true; +}; + +Core.checkSignature = function (Env, signedMsg, signature, publicKey) { + if (!(signedMsg && publicKey)) { return false; } + + var signedBuffer; + var pubBuffer; + var signatureBuffer; + + try { + signedBuffer = Nacl.util.decodeUTF8(signedMsg); + } catch (e) { + Env.Log.error('INVALID_SIGNED_BUFFER', signedMsg); + return null; + } + + try { + pubBuffer = Nacl.util.decodeBase64(publicKey); + } catch (e) { + return false; + } + + try { + signatureBuffer = Nacl.util.decodeBase64(signature); + } catch (e) { + return false; + } + + if (pubBuffer.length !== 32) { + Env.Log.error('PUBLIC_KEY_LENGTH', publicKey); + return false; + } + + if (signatureBuffer.length !== 64) { + return false; + } + + return Nacl.sign.detached.verify(signedBuffer, signatureBuffer, pubBuffer); +}; + +// E_NO_OWNERS +Core.hasOwners = function (metadata) { + return Boolean(metadata && Array.isArray(metadata.owners)); +}; + +Core.hasPendingOwners = function (metadata) { + return Boolean(metadata && Array.isArray(metadata.pending_owners)); +}; + +// INSUFFICIENT_PERMISSIONS +Core.isOwner = function (metadata, unsafeKey) { + return metadata.owners.indexOf(unsafeKey) !== -1; +}; + +Core.isPendingOwner = function (metadata, unsafeKey) { + return metadata.pending_owners.indexOf(unsafeKey) !== -1; +}; -// getChannelList -// getSession -// getHash -// getMultipleFileSize -// sumChannelSizes -// getFreeSpace -// getLimit diff --git a/lib/commands/pin-rpc.js b/lib/commands/pin-rpc.js index 3412f329e..16998b8b5 100644 --- a/lib/commands/pin-rpc.js +++ b/lib/commands/pin-rpc.js @@ -8,6 +8,8 @@ const Pinning = module.exports; const Nacl = require("tweetnacl/nacl-fast"); const Util = require("../common-util"); const nThen = require("nthen"); +const Saferphore = require("saferphore"); +const Pinned = require('../../scripts/pinned'); //const escapeKeyCharacters = Util.escapeKeyCharacters; const unescapeKeyCharacters = Util.unescapeKeyCharacters; @@ -397,3 +399,66 @@ Pinning.getFileSize = function (Env, channel, _cb) { }); }; +/* accepts a list, and returns a sublist of channel or file ids which seem + to have been deleted from the server (file size 0) + + we might consider that we should only say a file is gone if fs.stat returns + ENOENT, but for now it's simplest to just rely on getFileSize... +*/ +Pinning.getDeletedPads = function (Env, channels, cb) { + if (!Array.isArray(channels)) { return cb('INVALID_LIST'); } + var L = channels.length; + + var sem = Saferphore.create(10); + var absentees = []; + + var job = function (channel, wait) { + return function (give) { + Pinning.getFileSize(Env, channel, wait(give(function (e, size) { + if (e) { return; } + if (size === 0) { absentees.push(channel); } + }))); + }; + }; + + nThen(function (w) { + for (var i = 0; i < L; i++) { + sem.take(job(channels[i], w)); + } + }).nThen(function () { + cb(void 0, absentees); + }); +}; + +// inform that the +Pinning.loadChannelPins = function (Env) { + Pinned.load(function (err, data) { + if (err) { + Env.Log.error("LOAD_CHANNEL_PINS", err); + + // FIXME not sure what should be done here instead + Env.pinnedPads = {}; + Env.evPinnedPadsReady.fire(); + return; + } + + + Env.pinnedPads = data; + Env.evPinnedPadsReady.fire(); + }, { + pinPath: Env.paths.pin, + }); +}; + +Pinning.isChannelPinned = function (Env, channel, cb) { + Env.evPinnedPadsReady.reg(() => { + if (Env.pinnedPads[channel] && Object.keys(Env.pinnedPads[channel]).length) { + cb(true); + } else { + delete Env.pinnedPads[channel]; + cb(false); + } + }); +}; + + diff --git a/lib/commands/quota.js b/lib/commands/quota.js new file mode 100644 index 000000000..e7df14364 --- /dev/null +++ b/lib/commands/quota.js @@ -0,0 +1,111 @@ +/*jshint esversion: 6 */ +/* globals Buffer*/ +const Quota = module.exports; + +const Core = require("./commands/core"); +const Util = require("../common-util"); +const Package = require('../../package.json'); +const Https = require("https"); + +Quota.applyCustomLimits = function (Env, config) { + var isLimit = function (o) { + var valid = o && typeof(o) === 'object' && + typeof(o.limit) === 'number' && + typeof(o.plan) === 'string' && + typeof(o.note) === 'string'; + return valid; + }; + + // read custom limits from the config + var customLimits = (function (custom) { + var limits = {}; + Object.keys(custom).forEach(function (k) { + k.replace(/\/([^\/]+)$/, function (all, safeKey) { + var id = Util.unescapeKeyCharacters(safeKey || ''); + limits[id] = custom[k]; + return ''; + }); + }); + return limits; + }(config.customLimits || {})); + + Object.keys(customLimits).forEach(function (k) { + if (!isLimit(customLimits[k])) { return; } + Env.limits[k] = customLimits[k]; + }); +}; + +// The limits object contains storage limits for all the publicKey that have paid +// To each key is associated an object containing the 'limit' value and a 'note' explaining that limit +Quota.updateLimits = function (Env, config, publicKey, cb) { // FIXME BATCH?S + + if (config.adminEmail === false) { + Quota.applyCustomLimits(Env, config); + if (config.allowSubscriptions === false) { return; } + throw new Error("allowSubscriptions must be false if adminEmail is false"); + } + if (typeof cb !== "function") { cb = function () {}; } + + var defaultLimit = typeof(config.defaultStorageLimit) === 'number'? + config.defaultStorageLimit: Core.DEFAULT_LIMIT; + + var userId; + if (publicKey) { + userId = Util.unescapeKeyCharacters(publicKey); + } + + var body = JSON.stringify({ + domain: config.myDomain, + subdomain: config.mySubdomain || null, + adminEmail: config.adminEmail, + version: Package.version + }); + var options = { + host: 'accounts.cryptpad.fr', + path: '/api/getauthorized', + method: 'POST', + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(body) + } + }; + + var req = Https.request(options, function (response) { + if (!('' + response.statusCode).match(/^2\d\d$/)) { + return void cb('SERVER ERROR ' + response.statusCode); + } + var str = ''; + + response.on('data', function (chunk) { + str += chunk; + }); + + response.on('end', function () { + try { + var json = JSON.parse(str); + Env.limits = json; + Quota.applyCustomLimits(Env, config); + + var l; + if (userId) { + var limit = Env.limits[userId]; + l = limit && typeof limit.limit === "number" ? + [limit.limit, limit.plan, limit.note] : [defaultLimit, '', '']; + } + cb(void 0, l); + } catch (e) { + cb(e); + } + }); + }); + + req.on('error', function (e) { + Quota.applyCustomLimits(Env, config); + if (!config.domain) { return cb(); } + cb(e); + }); + + req.end(body); +}; + + diff --git a/lib/rpc.js b/lib/rpc.js index a064ebe64..cbaad1ee2 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -1,32 +1,26 @@ -/*@flow*/ /*jshint esversion: 6 */ /* Use Nacl for checking signatures of messages */ var Nacl = require("tweetnacl/nacl-fast"); /* globals Buffer*/ -/* globals process */ var Fs = require("fs"); var Fse = require("fs-extra"); var Path = require("path"); -var Https = require("https"); -const Package = require('../package.json'); -const Pinned = require('../scripts/pinned'); -const Saferphore = require("saferphore"); const nThen = require("nthen"); const Meta = require("./metadata"); const WriteQueue = require("./write-queue"); const BatchRead = require("./batch-read"); -const Core = require("./commands/core"); const Util = require("./common-util"); const escapeKeyCharacters = Util.escapeKeyCharacters; -const unescapeKeyCharacters = Util.unescapeKeyCharacters; const mkEvent = Util.mkEvent; +const Core = require("./commands/core"); const Admin = require("./commands/admin-rpc"); const Pinning = require("./commands/pin-rpc"); +const Quota = require("./commands/quota"); var RPC = module.exports; @@ -45,130 +39,6 @@ var WARN = function (e, output) { } }; -var makeCookie = function (token) { - var time = (+new Date()); - time -= time % 5000; - - return [ - time, - process.pid, - token - ]; -}; - -var parseCookie = function (cookie) { - if (!(cookie && cookie.split)) { return null; } - - var parts = cookie.split('|'); - if (parts.length !== 3) { return null; } - - var c = {}; - c.time = new Date(parts[0]); - c.pid = Number(parts[1]); - c.seq = parts[2]; - return c; -}; - -var isTooOld = function (time, now) { - return (now - time) > 300000; -}; - -var expireSession = function (Sessions, key) { - var session = Sessions[key]; - if (!session) { return; } - if (session.blobstage) { - session.blobstage.close(); - } - delete Sessions[key]; -}; - -var expireSessions = function (Sessions) { - var now = +new Date(); - Object.keys(Sessions).forEach(function (key) { - var session = Sessions[key]; - if (session && isTooOld(session.atime, now)) { - expireSession(Sessions, key); - } - }); -}; - -var addTokenForKey = function (Sessions, publicKey, token) { - if (!Sessions[publicKey]) { throw new Error('undefined user'); } - - var user = Core.getSession(Sessions, publicKey); - user.tokens.push(token); - user.atime = +new Date(); - if (user.tokens.length > 2) { user.tokens.shift(); } -}; - -var isValidCookie = function (Sessions, publicKey, cookie) { - var parsed = parseCookie(cookie); - if (!parsed) { return false; } - - var now = +new Date(); - - if (!parsed.time) { return false; } - if (isTooOld(parsed.time, now)) { - return false; - } - - // different process. try harder - if (process.pid !== parsed.pid) { - return false; - } - - var user = Core.getSession(Sessions, publicKey); - if (!user) { return false; } - - var idx = user.tokens.indexOf(parsed.seq); - if (idx === -1) { return false; } - - if (idx > 0) { - // make a new token - addTokenForKey(Sessions, publicKey, Core.makeToken()); - } - - return true; -}; - -var checkSignature = function (signedMsg, signature, publicKey) { - if (!(signedMsg && publicKey)) { return false; } - - var signedBuffer; - var pubBuffer; - var signatureBuffer; - - try { - signedBuffer = Nacl.util.decodeUTF8(signedMsg); - } catch (e) { - Log.error('INVALID_SIGNED_BUFFER', signedMsg); - return null; - } - - try { - pubBuffer = Nacl.util.decodeBase64(publicKey); - } catch (e) { - return false; - } - - try { - signatureBuffer = Nacl.util.decodeBase64(signature); - } catch (e) { - return false; - } - - if (pubBuffer.length !== 32) { - Log.error('PUBLIC_KEY_LENGTH', publicKey); - return false; - } - - if (signatureBuffer.length !== 64) { - return false; - } - - return Nacl.sign.detached.verify(signedBuffer, signatureBuffer, pubBuffer); -}; - const batchMetadata = BatchRead("GET_METADATA"); var getMetadata = function (Env, channel, cb) { if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } @@ -188,24 +58,6 @@ var getMetadata = function (Env, channel, cb) { }); }; -// E_NO_OWNERS -var hasOwners = function (metadata) { - return Boolean(metadata && Array.isArray(metadata.owners)); -}; - -var hasPendingOwners = function (metadata) { - return Boolean(metadata && Array.isArray(metadata.pending_owners)); -}; - -// INSUFFICIENT_PERMISSIONS -var isOwner = function (metadata, unsafeKey) { - return metadata.owners.indexOf(unsafeKey) !== -1; -}; - -var isPendingOwner = function (metadata, unsafeKey) { - return metadata.pending_owners.indexOf(unsafeKey) !== -1; -}; - /* setMetadata - write a new line to the metadata log if a valid command is provided - data is an object: { @@ -228,7 +80,7 @@ var setMetadata = function (Env, data, unsafeKey, cb) { cb(err); return void next(); } - if (!hasOwners(metadata)) { + if (!Core.hasOwners(metadata)) { cb('E_NO_OWNERS'); return void next(); } @@ -243,9 +95,9 @@ var setMetadata = function (Env, data, unsafeKey, cb) { // Confirm that the channel is owned by the user in question // or the user is accepting a pending ownership offer - if (hasPendingOwners(metadata) && - isPendingOwner(metadata, unsafeKey) && - !isOwner(metadata, unsafeKey)) { + if (Core.hasPendingOwners(metadata) && + Core.isPendingOwner(metadata, unsafeKey) && + !Core.isOwner(metadata, unsafeKey)) { // If you are a pending owner, make sure you can only add yourelf as an owner if ((command !== 'ADD_OWNERS' && command !== 'RM_PENDING_OWNERS') @@ -258,7 +110,7 @@ var setMetadata = function (Env, data, unsafeKey, cb) { // FIXME wacky fallthrough is hard to read // we could pass this off to a writeMetadataCommand function // and make the flow easier to follow - } else if (!isOwner(metadata, unsafeKey)) { + } else if (!Core.isOwner(metadata, unsafeKey)) { cb('INSUFFICIENT_PERMISSIONS'); return void next(); } @@ -291,169 +143,6 @@ var setMetadata = function (Env, data, unsafeKey, cb) { }); }; -/* accepts a list, and returns a sublist of channel or file ids which seem - to have been deleted from the server (file size 0) - - we might consider that we should only say a file is gone if fs.stat returns - ENOENT, but for now it's simplest to just rely on getFileSize... -*/ -var getDeletedPads = function (Env, channels, cb) { - if (!Array.isArray(channels)) { return cb('INVALID_LIST'); } - var L = channels.length; - - var sem = Saferphore.create(10); - var absentees = []; - - var job = function (channel, wait) { - return function (give) { - Pinning.getFileSize(Env, channel, wait(give(function (e, size) { - if (e) { return; } - if (size === 0) { absentees.push(channel); } - }))); - }; - }; - - nThen(function (w) { - for (var i = 0; i < L; i++) { - sem.take(job(channels[i], w)); - } - }).nThen(function () { - cb(void 0, absentees); - }); -}; - -var applyCustomLimits = function (Env, config) { - var isLimit = function (o) { - var valid = o && typeof(o) === 'object' && - typeof(o.limit) === 'number' && - typeof(o.plan) === 'string' && - typeof(o.note) === 'string'; - return valid; - }; - - // read custom limits from the config - var customLimits = (function (custom) { - var limits = {}; - Object.keys(custom).forEach(function (k) { - k.replace(/\/([^\/]+)$/, function (all, safeKey) { - var id = unescapeKeyCharacters(safeKey || ''); - limits[id] = custom[k]; - return ''; - }); - }); - return limits; - }(config.customLimits || {})); - - Object.keys(customLimits).forEach(function (k) { - if (!isLimit(customLimits[k])) { return; } - Env.limits[k] = customLimits[k]; - }); -}; - -// The limits object contains storage limits for all the publicKey that have paid -// To each key is associated an object containing the 'limit' value and a 'note' explaining that limit -var updateLimits = function (Env, config, publicKey, cb /*:(?string, ?any[])=>void*/) { // FIXME BATCH? - - if (config.adminEmail === false) { - applyCustomLimits(Env, config); - if (config.allowSubscriptions === false) { return; } - throw new Error("allowSubscriptions must be false if adminEmail is false"); - } - if (typeof cb !== "function") { cb = function () {}; } - - var defaultLimit = typeof(config.defaultStorageLimit) === 'number'? - config.defaultStorageLimit: Core.DEFAULT_LIMIT; - - var userId; - if (publicKey) { - userId = unescapeKeyCharacters(publicKey); - } - - var body = JSON.stringify({ - domain: config.myDomain, - subdomain: config.mySubdomain || null, - adminEmail: config.adminEmail, - version: Package.version - }); - var options = { - host: 'accounts.cryptpad.fr', - path: '/api/getauthorized', - method: 'POST', - headers: { - "Content-Type": "application/json", - "Content-Length": Buffer.byteLength(body) - } - }; - - var req = Https.request(options, function (response) { - if (!('' + response.statusCode).match(/^2\d\d$/)) { - return void cb('SERVER ERROR ' + response.statusCode); - } - var str = ''; - - response.on('data', function (chunk) { - str += chunk; - }); - - response.on('end', function () { - try { - var json = JSON.parse(str); - Env.limits = json; - applyCustomLimits(Env, config); - - var l; - if (userId) { - var limit = Env.limits[userId]; - l = limit && typeof limit.limit === "number" ? - [limit.limit, limit.plan, limit.note] : [defaultLimit, '', '']; - } - cb(void 0, l); - } catch (e) { - cb(e); - } - }); - }); - - req.on('error', function (e) { - applyCustomLimits(Env, config); - if (!config.domain) { return cb(); } - cb(e); - }); - - req.end(body); -}; - -// inform that the -var loadChannelPins = function (Env) { - Pinned.load(function (err, data) { - if (err) { - Log.error("LOAD_CHANNEL_PINS", err); - - // FIXME not sure what should be done here instead - Env.pinnedPads = {}; - Env.evPinnedPadsReady.fire(); - return; - } - - - Env.pinnedPads = data; - Env.evPinnedPadsReady.fire(); - }, { - pinPath: Env.paths.pin, - }); -}; - -var isChannelPinned = function (Env, channel, cb) { - Env.evPinnedPadsReady.reg(() => { - if (Env.pinnedPads[channel] && Object.keys(Env.pinnedPads[channel]).length) { - cb(true); - } else { - delete Env.pinnedPads[channel]; - cb(false); - } - }); -}; - var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { if (typeof(channelId) !== 'string' || channelId.length !== 32) { return cb('INVALID_ARGUMENTS'); @@ -461,9 +150,9 @@ var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } - if (!hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } + if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } // Confirm that the channel is owned by the user in question - if (!isOwner(metadata, unsafeKey)) { + if (!Core.isOwner(metadata, unsafeKey)) { return void cb('INSUFFICIENT_PERMISSIONS'); } return void Env.msgStore.clearChannel(channelId, function (e) { @@ -520,8 +209,8 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } - if (!hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } - if (!isOwner(metadata, unsafeKey)) { + if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } + if (!Core.isOwner(metadata, unsafeKey)) { return void cb('INSUFFICIENT_PERMISSIONS'); } // temporarily archive the file @@ -540,11 +229,11 @@ var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { nThen(function (w) { getMetadata(Env, channelId, w(function (err, metadata) { if (err) { return void cb(err); } - if (!hasOwners(metadata)) { + if (!Core.hasOwners(metadata)) { w.abort(); return void cb('E_NO_OWNERS'); } - if (!isOwner(metadata, unsafeKey)) { + if (!Core.isOwner(metadata, unsafeKey)) { w.abort(); return void cb("INSUFFICIENT_PERMISSIONS"); } @@ -956,7 +645,7 @@ RPC.create = function (config, cb) { respond(e, [null, dict, null]); }); case 'GET_DELETED_PADS': - return void getDeletedPads(Env, msg[1], function (e, list) { + return void Pinning.getDeletedPads(Env, msg[1], function (e, list) { if (e) { WARN(e, msg[1]); return respond(e); @@ -964,7 +653,7 @@ RPC.create = function (config, cb) { respond(e, [null, list, null]); }); case 'IS_CHANNEL_PINNED': - return void isChannelPinned(Env, msg[1], function (isPinned) { + return void Pinning.isChannelPinned(Env, msg[1], function (isPinned) { respond(null, [null, isPinned, null]); }); case 'IS_NEW_CHANNEL': @@ -1014,7 +703,7 @@ RPC.create = function (config, cb) { } var cookie = msg[0]; - if (!isValidCookie(Sessions, publicKey, cookie)) { + if (!Core.isValidCookie(Sessions, publicKey, cookie)) { // no cookie is fine if the RPC is to get a cookie if (msg[1] !== 'COOKIE') { return void respond('NO_COOKIE'); @@ -1028,7 +717,7 @@ RPC.create = function (config, cb) { } if (isAuthenticatedCall(msg[1])) { - if (checkSignature(serialized, signature, publicKey) !== true) { + if (Core.checkSignature(Env, serialized, signature, publicKey) !== true) { return void respond("INVALID_SIGNATURE_OR_PUBLIC_KEY"); } } else if (msg[1] !== 'UPLOAD') { @@ -1052,7 +741,7 @@ RPC.create = function (config, cb) { var Respond = function (e, msg) { var session = Sessions[safeKey]; var token = session? session.tokens.slice(-1)[0]: ''; - var cookie = makeCookie(token).join('|'); + var cookie = Core.makeCookie(token).join('|'); respond(e ? String(e): e, [cookie].concat(typeof(msg) !== 'undefined' ?msg: [])); }; @@ -1093,7 +782,7 @@ RPC.create = function (config, cb) { Respond(e, size); }); case 'UPDATE_LIMITS': - return void updateLimits(Env, config, safeKey, function (e, limit) { + return void Quota.updateLimits(Env, config, safeKey, function (e, limit) { if (e) { WARN(e, limit); return void Respond(e); @@ -1110,7 +799,7 @@ RPC.create = function (config, cb) { }); case 'EXPIRE_SESSION': return void setTimeout(function () { - expireSession(Sessions, safeKey); + Core.expireSession(Sessions, safeKey); Respond(void 0, "OK"); }); case 'CLEAR_OWNED_CHANNEL': @@ -1223,17 +912,17 @@ RPC.create = function (config, cb) { }; var updateLimitDaily = function () { - updateLimits(Env, config, undefined, function (e) { + Quota.updateLimits(Env, config, undefined, function (e) { if (e) { WARN('limitUpdate', e); } }); }; - applyCustomLimits(Env, config); + Quota.applyCustomLimits(Env, config); updateLimitDaily(); setInterval(updateLimitDaily, 24*3600*1000); - loadChannelPins(Env); + Pinning.loadChannelPins(Env); nThen(function (w) { Store.create({ @@ -1257,7 +946,7 @@ RPC.create = function (config, cb) { // expire old sessions once per minute // XXX allow for graceful shutdown Env.sessionExpirationInterval = setInterval(function () { - expireSessions(Sessions); + Core.expireSessions(Sessions); }, Core.SESSION_EXPIRATION_TIME); }); }; From bb7e8e4512108bd520ff00b9e6fd737221b095ad Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 13:14:26 -0500 Subject: [PATCH 25/53] move login block functionality into its own rpc module --- lib/commands/block.js | 172 +++++++++++++++++++++++++++++++++++++++++ lib/rpc.js | 175 +----------------------------------------- 2 files changed, 175 insertions(+), 172 deletions(-) create mode 100644 lib/commands/block.js diff --git a/lib/commands/block.js b/lib/commands/block.js new file mode 100644 index 000000000..90837745e --- /dev/null +++ b/lib/commands/block.js @@ -0,0 +1,172 @@ +/*jshint esversion: 6 */ +/* globals Buffer*/ +var Block = module.exports; + +const Fs = require("fs"); +const Fse = require("fs-extra"); +const Path = require("path"); +const Nacl = require("tweetnacl/nacl-fast"); +const nThen = require("nthen"); + +const Util = require("../common-util"); + +/* + We assume that the server is secured against MitM attacks + via HTTPS, and that malicious actors do not have code execution + capabilities. If they do, we have much more serious problems. + + The capability to replay a block write or remove results in either + a denial of service for the user whose block was removed, or in the + case of a write, a rollback to an earlier password. + + Since block modification is destructive, this can result in loss + of access to the user's drive. + + So long as the detached signature is never observed by a malicious + party, and the server discards it after proof of knowledge, replays + are not possible. However, this precludes verification of the signature + at a later time. + + Despite this, an integrity check is still possible by the original + author of the block, since we assume that the block will have been + encrypted with xsalsa20-poly1305 which is authenticated. +*/ +Block.validateLoginBlock = function (Env, publicKey, signature, block, cb) { // FIXME BLOCKS + // convert the public key to a Uint8Array and validate it + if (typeof(publicKey) !== 'string') { return void cb('E_INVALID_KEY'); } + + var u8_public_key; + try { + u8_public_key = Nacl.util.decodeBase64(publicKey); + } catch (e) { + return void cb('E_INVALID_KEY'); + } + + var u8_signature; + try { + u8_signature = Nacl.util.decodeBase64(signature); + } catch (e) { + Env.Log.error('INVALID_BLOCK_SIGNATURE', e); + return void cb('E_INVALID_SIGNATURE'); + } + + // convert the block to a Uint8Array + var u8_block; + try { + u8_block = Nacl.util.decodeBase64(block); + } catch (e) { + return void cb('E_INVALID_BLOCK'); + } + + // take its hash + var hash = Nacl.hash(u8_block); + + // validate the signature against the hash of the content + var verified = Nacl.sign.detached.verify(hash, u8_signature, u8_public_key); + + // existing authentication ensures that users cannot replay old blocks + + // call back with (err) if unsuccessful + if (!verified) { return void cb("E_COULD_NOT_VERIFY"); } + + return void cb(null, u8_block); +}; + +var createLoginBlockPath = function (Env, publicKey) { // FIXME BLOCKS + // prepare publicKey to be used as a file name + var safeKey = Util.escapeKeyCharacters(publicKey); + + // validate safeKey + if (typeof(safeKey) !== 'string') { + return; + } + + // derive the full path + // /home/cryptpad/cryptpad/block/fg/fg32kefksjdgjkewrjksdfksjdfsdfskdjfsfd + return Path.join(Env.paths.block, safeKey.slice(0, 2), safeKey); +}; + +Block.writeLoginBlock = function (Env, msg, cb) { // FIXME BLOCKS + //console.log(msg); + var publicKey = msg[0]; + var signature = msg[1]; + var block = msg[2]; + + Block.validateLoginBlock(Env, publicKey, signature, block, function (e, validatedBlock) { + if (e) { return void cb(e); } + if (!(validatedBlock instanceof Uint8Array)) { return void cb('E_INVALID_BLOCK'); } + + // derive the filepath + var path = createLoginBlockPath(Env, publicKey); + + // make sure the path is valid + if (typeof(path) !== 'string') { + return void cb('E_INVALID_BLOCK_PATH'); + } + + var parsed = Path.parse(path); + if (!parsed || typeof(parsed.dir) !== 'string') { + return void cb("E_INVALID_BLOCK_PATH_2"); + } + + nThen(function (w) { + // make sure the path to the file exists + Fse.mkdirp(parsed.dir, w(function (e) { + if (e) { + w.abort(); + cb(e); + } + })); + }).nThen(function () { + // actually write the block + + // flow is dumb and I need to guard against this which will never happen + /*:: if (typeof(validatedBlock) === 'undefined') { throw new Error('should never happen'); } */ + /*:: if (typeof(path) === 'undefined') { throw new Error('should never happen'); } */ + Fs.writeFile(path, Buffer.from(validatedBlock), { encoding: "binary", }, function (err) { + if (err) { return void cb(err); } + cb(); + }); + }); + }); +}; + +/* + When users write a block, they upload the block, and provide + a signature proving that they deserve to be able to write to + the location determined by the public key. + + When removing a block, there is nothing to upload, but we need + to sign something. Since the signature is considered sensitive + information, we can just sign some constant and use that as proof. + +*/ +Block.removeLoginBlock = function (Env, msg, cb) { // FIXME BLOCKS + var publicKey = msg[0]; + var signature = msg[1]; + var block = Nacl.util.decodeUTF8('DELETE_BLOCK'); // clients and the server will have to agree on this constant + + Block.validateLoginBlock(Env, publicKey, signature, block, function (e /*::, validatedBlock */) { + if (e) { return void cb(e); } + // derive the filepath + var path = createLoginBlockPath(Env, publicKey); + + // make sure the path is valid + if (typeof(path) !== 'string') { + return void cb('E_INVALID_BLOCK_PATH'); + } + + // FIXME COLDSTORAGE + Fs.unlink(path, function (err) { + Env.Log.info('DELETION_BLOCK_BY_OWNER_RPC', { + publicKey: publicKey, + path: path, + status: err? String(err): 'SUCCESS', + }); + + if (err) { return void cb(err); } + cb(); + }); + }); +}; + diff --git a/lib/rpc.js b/lib/rpc.js index cbaad1ee2..3785b20a1 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -1,13 +1,4 @@ /*jshint esversion: 6 */ -/* Use Nacl for checking signatures of messages */ -var Nacl = require("tweetnacl/nacl-fast"); - -/* globals Buffer*/ - -var Fs = require("fs"); - -var Fse = require("fs-extra"); -var Path = require("path"); const nThen = require("nthen"); const Meta = require("./metadata"); const WriteQueue = require("./write-queue"); @@ -21,6 +12,7 @@ const Core = require("./commands/core"); const Admin = require("./commands/admin-rpc"); const Pinning = require("./commands/pin-rpc"); const Quota = require("./commands/quota"); +const Block = require("./commands/block"); var RPC = module.exports; @@ -249,167 +241,6 @@ var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { }); }; - -/* - We assume that the server is secured against MitM attacks - via HTTPS, and that malicious actors do not have code execution - capabilities. If they do, we have much more serious problems. - - The capability to replay a block write or remove results in either - a denial of service for the user whose block was removed, or in the - case of a write, a rollback to an earlier password. - - Since block modification is destructive, this can result in loss - of access to the user's drive. - - So long as the detached signature is never observed by a malicious - party, and the server discards it after proof of knowledge, replays - are not possible. However, this precludes verification of the signature - at a later time. - - Despite this, an integrity check is still possible by the original - author of the block, since we assume that the block will have been - encrypted with xsalsa20-poly1305 which is authenticated. -*/ -var validateLoginBlock = function (Env, publicKey, signature, block, cb) { // FIXME BLOCKS - // convert the public key to a Uint8Array and validate it - if (typeof(publicKey) !== 'string') { return void cb('E_INVALID_KEY'); } - - var u8_public_key; - try { - u8_public_key = Nacl.util.decodeBase64(publicKey); - } catch (e) { - return void cb('E_INVALID_KEY'); - } - - var u8_signature; - try { - u8_signature = Nacl.util.decodeBase64(signature); - } catch (e) { - Log.error('INVALID_BLOCK_SIGNATURE', e); - return void cb('E_INVALID_SIGNATURE'); - } - - // convert the block to a Uint8Array - var u8_block; - try { - u8_block = Nacl.util.decodeBase64(block); - } catch (e) { - return void cb('E_INVALID_BLOCK'); - } - - // take its hash - var hash = Nacl.hash(u8_block); - - // validate the signature against the hash of the content - var verified = Nacl.sign.detached.verify(hash, u8_signature, u8_public_key); - - // existing authentication ensures that users cannot replay old blocks - - // call back with (err) if unsuccessful - if (!verified) { return void cb("E_COULD_NOT_VERIFY"); } - - return void cb(null, u8_block); -}; - -var createLoginBlockPath = function (Env, publicKey) { // FIXME BLOCKS - // prepare publicKey to be used as a file name - var safeKey = escapeKeyCharacters(publicKey); - - // validate safeKey - if (typeof(safeKey) !== 'string') { - return; - } - - // derive the full path - // /home/cryptpad/cryptpad/block/fg/fg32kefksjdgjkewrjksdfksjdfsdfskdjfsfd - return Path.join(Env.paths.block, safeKey.slice(0, 2), safeKey); -}; - -var writeLoginBlock = function (Env, msg, cb) { // FIXME BLOCKS - //console.log(msg); - var publicKey = msg[0]; - var signature = msg[1]; - var block = msg[2]; - - validateLoginBlock(Env, publicKey, signature, block, function (e, validatedBlock) { - if (e) { return void cb(e); } - if (!(validatedBlock instanceof Uint8Array)) { return void cb('E_INVALID_BLOCK'); } - - // derive the filepath - var path = createLoginBlockPath(Env, publicKey); - - // make sure the path is valid - if (typeof(path) !== 'string') { - return void cb('E_INVALID_BLOCK_PATH'); - } - - var parsed = Path.parse(path); - if (!parsed || typeof(parsed.dir) !== 'string') { - return void cb("E_INVALID_BLOCK_PATH_2"); - } - - nThen(function (w) { - // make sure the path to the file exists - Fse.mkdirp(parsed.dir, w(function (e) { - if (e) { - w.abort(); - cb(e); - } - })); - }).nThen(function () { - // actually write the block - - // flow is dumb and I need to guard against this which will never happen - /*:: if (typeof(validatedBlock) === 'undefined') { throw new Error('should never happen'); } */ - /*:: if (typeof(path) === 'undefined') { throw new Error('should never happen'); } */ - Fs.writeFile(path, Buffer.from(validatedBlock), { encoding: "binary", }, function (err) { - if (err) { return void cb(err); } - cb(); - }); - }); - }); -}; - -/* - When users write a block, they upload the block, and provide - a signature proving that they deserve to be able to write to - the location determined by the public key. - - When removing a block, there is nothing to upload, but we need - to sign something. Since the signature is considered sensitive - information, we can just sign some constant and use that as proof. - -*/ -var removeLoginBlock = function (Env, msg, cb) { // FIXME BLOCKS - var publicKey = msg[0]; - var signature = msg[1]; - var block = Nacl.util.decodeUTF8('DELETE_BLOCK'); // clients and the server will have to agree on this constant - - validateLoginBlock(Env, publicKey, signature, block, function (e /*::, validatedBlock */) { - if (e) { return void cb(e); } - // derive the filepath - var path = createLoginBlockPath(Env, publicKey); - - // make sure the path is valid - if (typeof(path) !== 'string') { - return void cb('E_INVALID_BLOCK_PATH'); - } - - // FIXME COLDSTORAGE - Fs.unlink(path, function (err) { - Log.info('DELETION_BLOCK_BY_OWNER_RPC', { - publicKey: publicKey, - path: path, - status: err? String(err): 'SUCCESS', - }); - - if (err) { return void cb(err); } - cb(); - }); - }); -}; - var ARRAY_LINE = /^\[/; /* Files can contain metadata but not content @@ -863,7 +694,7 @@ RPC.create = function (config, cb) { Respond(e); }); case 'WRITE_LOGIN_BLOCK': - return void writeLoginBlock(Env, msg[1], function (e) { + return void Block.writeLoginBlock(Env, msg[1], function (e) { if (e) { WARN(e, 'WRITE_LOGIN_BLOCK'); return void Respond(e); @@ -871,7 +702,7 @@ RPC.create = function (config, cb) { Respond(e); }); case 'REMOVE_LOGIN_BLOCK': - return void removeLoginBlock(Env, msg[1], function (e) { + return void Block.removeLoginBlock(Env, msg[1], function (e) { if (e) { WARN(e, 'REMOVE_LOGIN_BLOCK'); return void Respond(e); From c1f222dd6c8084790efea2313f86934270bbc672 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 13:19:40 -0500 Subject: [PATCH 26/53] move metadata commands from rpc to their own module --- lib/commands/metadata.js | 113 +++++++++++++++++++++++++++++++++++++ lib/rpc.js | 118 ++------------------------------------- 2 files changed, 119 insertions(+), 112 deletions(-) create mode 100644 lib/commands/metadata.js diff --git a/lib/commands/metadata.js b/lib/commands/metadata.js new file mode 100644 index 000000000..91b08d8cc --- /dev/null +++ b/lib/commands/metadata.js @@ -0,0 +1,113 @@ +/*jshint esversion: 6 */ +const Data = module.exports; + +const Meta = require("../metadata"); +const BatchRead = require("../batch-read"); +const WriteQueue = require("./write-queue"); +const Core = require("./commands/core"); + +const batchMetadata = BatchRead("GET_METADATA"); +Data.getMetadata = function (Env, channel, cb) { + if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } + if (channel.length !== 32) { return cb("INVALID_CHAN_LENGTH"); } + + batchMetadata(channel, cb, function (done) { + var ref = {}; + var lineHandler = Meta.createLineHandler(ref, Env.Log.error); + + return void Env.msgStore.readChannelMetadata(channel, lineHandler, function (err) { + if (err) { + // stream errors? + return void done(err); + } + done(void 0, ref.meta); + }); + }); +}; + +/* setMetadata + - write a new line to the metadata log if a valid command is provided + - data is an object: { + channel: channelId, + command: metadataCommand (string), + value: value + } +*/ +var queueMetadata = WriteQueue(); +Data.setMetadata = function (Env, data, unsafeKey, cb) { + var channel = data.channel; + var command = data.command; + if (!channel || !Core.isValidId(channel)) { return void cb ('INVALID_CHAN'); } + if (!command || typeof (command) !== 'string') { return void cb ('INVALID_COMMAND'); } + if (Meta.commands.indexOf(command) === -1) { return void('UNSUPPORTED_COMMAND'); } + + queueMetadata(channel, function (next) { + Data.getMetadata(Env, channel, function (err, metadata) { + if (err) { + cb(err); + return void next(); + } + if (!Core.hasOwners(metadata)) { + cb('E_NO_OWNERS'); + return void next(); + } + + // if you are a pending owner and not an owner + // you can either ADD_OWNERS, or RM_PENDING_OWNERS + // and you should only be able to add yourself as an owner + // everything else should be rejected + // else if you are not an owner + // you should be rejected + // else write the command + + // Confirm that the channel is owned by the user in question + // or the user is accepting a pending ownership offer + if (Core.hasPendingOwners(metadata) && + Core.isPendingOwner(metadata, unsafeKey) && + !Core.isOwner(metadata, unsafeKey)) { + + // If you are a pending owner, make sure you can only add yourelf as an owner + if ((command !== 'ADD_OWNERS' && command !== 'RM_PENDING_OWNERS') + || !Array.isArray(data.value) + || data.value.length !== 1 + || data.value[0] !== unsafeKey) { + cb('INSUFFICIENT_PERMISSIONS'); + return void next(); + } + // FIXME wacky fallthrough is hard to read + // we could pass this off to a writeMetadataCommand function + // and make the flow easier to follow + } else if (!Core.isOwner(metadata, unsafeKey)) { + cb('INSUFFICIENT_PERMISSIONS'); + return void next(); + } + + // Add the new metadata line + var line = [command, data.value, +new Date()]; + var changed = false; + try { + changed = Meta.handleCommand(metadata, line); + } catch (e) { + cb(e); + return void next(); + } + + // if your command is valid but it didn't result in any change to the metadata, + // call back now and don't write any "useless" line to the log + if (!changed) { + cb(void 0, metadata); + return void next(); + } + Env.msgStore.writeMetadata(channel, JSON.stringify(line), function (e) { + if (e) { + cb(e); + return void next(); + } + cb(void 0, metadata); + next(); + }); + }); + }); +}; + + diff --git a/lib/rpc.js b/lib/rpc.js index 3785b20a1..0ddfdb128 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -1,8 +1,5 @@ /*jshint esversion: 6 */ const nThen = require("nthen"); -const Meta = require("./metadata"); -const WriteQueue = require("./write-queue"); -const BatchRead = require("./batch-read"); const Util = require("./common-util"); const escapeKeyCharacters = Util.escapeKeyCharacters; @@ -13,6 +10,7 @@ const Admin = require("./commands/admin-rpc"); const Pinning = require("./commands/pin-rpc"); const Quota = require("./commands/quota"); const Block = require("./commands/block"); +const Metadata = require("./commands/metadata"); var RPC = module.exports; @@ -31,116 +29,12 @@ var WARN = function (e, output) { } }; -const batchMetadata = BatchRead("GET_METADATA"); -var getMetadata = function (Env, channel, cb) { - if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } - if (channel.length !== 32) { return cb("INVALID_CHAN_LENGTH"); } - - batchMetadata(channel, cb, function (done) { - var ref = {}; - var lineHandler = Meta.createLineHandler(ref, Log.error); - - return void Env.msgStore.readChannelMetadata(channel, lineHandler, function (err) { - if (err) { - // stream errors? - return void done(err); - } - done(void 0, ref.meta); - }); - }); -}; - -/* setMetadata - - write a new line to the metadata log if a valid command is provided - - data is an object: { - channel: channelId, - command: metadataCommand (string), - value: value - } -*/ -var queueMetadata = WriteQueue(); -var setMetadata = function (Env, data, unsafeKey, cb) { - var channel = data.channel; - var command = data.command; - if (!channel || !Core.isValidId(channel)) { return void cb ('INVALID_CHAN'); } - if (!command || typeof (command) !== 'string') { return void cb ('INVALID_COMMAND'); } - if (Meta.commands.indexOf(command) === -1) { return void('UNSUPPORTED_COMMAND'); } - - queueMetadata(channel, function (next) { - getMetadata(Env, channel, function (err, metadata) { - if (err) { - cb(err); - return void next(); - } - if (!Core.hasOwners(metadata)) { - cb('E_NO_OWNERS'); - return void next(); - } - - // if you are a pending owner and not an owner - // you can either ADD_OWNERS, or RM_PENDING_OWNERS - // and you should only be able to add yourself as an owner - // everything else should be rejected - // else if you are not an owner - // you should be rejected - // else write the command - - // Confirm that the channel is owned by the user in question - // or the user is accepting a pending ownership offer - if (Core.hasPendingOwners(metadata) && - Core.isPendingOwner(metadata, unsafeKey) && - !Core.isOwner(metadata, unsafeKey)) { - - // If you are a pending owner, make sure you can only add yourelf as an owner - if ((command !== 'ADD_OWNERS' && command !== 'RM_PENDING_OWNERS') - || !Array.isArray(data.value) - || data.value.length !== 1 - || data.value[0] !== unsafeKey) { - cb('INSUFFICIENT_PERMISSIONS'); - return void next(); - } - // FIXME wacky fallthrough is hard to read - // we could pass this off to a writeMetadataCommand function - // and make the flow easier to follow - } else if (!Core.isOwner(metadata, unsafeKey)) { - cb('INSUFFICIENT_PERMISSIONS'); - return void next(); - } - - // Add the new metadata line - var line = [command, data.value, +new Date()]; - var changed = false; - try { - changed = Meta.handleCommand(metadata, line); - } catch (e) { - cb(e); - return void next(); - } - - // if your command is valid but it didn't result in any change to the metadata, - // call back now and don't write any "useless" line to the log - if (!changed) { - cb(void 0, metadata); - return void next(); - } - Env.msgStore.writeMetadata(channel, JSON.stringify(line), function (e) { - if (e) { - cb(e); - return void next(); - } - cb(void 0, metadata); - next(); - }); - }); - }); -}; - var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { if (typeof(channelId) !== 'string' || channelId.length !== 32) { return cb('INVALID_ARGUMENTS'); } - getMetadata(Env, channelId, function (err, metadata) { + Metadata.getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } // Confirm that the channel is owned by the user in question @@ -199,7 +93,7 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { }); } - getMetadata(Env, channelId, function (err, metadata) { + Metadata.getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } if (!Core.isOwner(metadata, unsafeKey)) { @@ -219,7 +113,7 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { nThen(function (w) { - getMetadata(Env, channelId, w(function (err, metadata) { + Metadata.getMetadata(Env, channelId, w(function (err, metadata) { if (err) { return void cb(err); } if (!Core.hasOwners(metadata)) { w.abort(); @@ -463,7 +357,7 @@ RPC.create = function (config, cb) { respond(e, [null, size, null]); }); case 'GET_METADATA': - return void getMetadata(Env, msg[1], function (e, data) { + return void Metadata.getMetadata(Env, msg[1], function (e, data) { WARN(e, msg[1]); respond(e, [null, data, null]); }); @@ -718,7 +612,7 @@ RPC.create = function (config, cb) { Respond(void 0, result); }); case 'SET_METADATA': - return void setMetadata(Env, msg[1], publicKey, function (e, data) { + return void Metadata.setMetadata(Env, msg[1], publicKey, function (e, data) { if (e) { WARN(e, data); return void Respond(e); From ceb351326c1b50146f175a9ea8010333c97acb82 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 13:36:14 -0500 Subject: [PATCH 27/53] split out some more rpc functionality and fix broken module paths --- lib/commands/channel.js | 193 +++++++++++++++++++++++++++++ lib/commands/metadata.js | 4 +- lib/commands/quota.js | 2 +- lib/commands/upload.js | 37 ++++++ lib/rpc.js | 260 ++++----------------------------------- 5 files changed, 255 insertions(+), 241 deletions(-) create mode 100644 lib/commands/channel.js create mode 100644 lib/commands/upload.js diff --git a/lib/commands/channel.js b/lib/commands/channel.js new file mode 100644 index 000000000..052aa3c44 --- /dev/null +++ b/lib/commands/channel.js @@ -0,0 +1,193 @@ +/*jshint esversion: 6 */ +const Channel = module.exports; + +const Util = require("../common-util"); +const nThen = require("nthen"); +const Core = require("./core"); +const Metadata = require("./metadata"); + +Channel.clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { + if (typeof(channelId) !== 'string' || channelId.length !== 32) { + return cb('INVALID_ARGUMENTS'); + } + + Metadata.getMetadata(Env, channelId, function (err, metadata) { + if (err) { return void cb(err); } + if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } + // Confirm that the channel is owned by the user in question + if (!Core.isOwner(metadata, unsafeKey)) { + return void cb('INSUFFICIENT_PERMISSIONS'); + } + return void Env.msgStore.clearChannel(channelId, function (e) { + cb(e); + }); + }); +}; + +Channel.removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { + if (typeof(channelId) !== 'string' || !Core.isValidId(channelId)) { + return cb('INVALID_ARGUMENTS'); + } + + if (Env.blobStore.isFileId(channelId)) { + var safeKey = Util.escapeKeyCharacters(unsafeKey); + var blobId = channelId; + + return void nThen(function (w) { + // check if you have permissions + Env.blobStore.isOwnedBy(safeKey, blobId, w(function (err, owned) { + if (err || !owned) { + w.abort(); + return void cb("INSUFFICIENT_PERMISSIONS"); + } + })); + }).nThen(function (w) { + // remove the blob + return void Env.blobStore.archive.blob(blobId, w(function (err) { + Env.Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', { + safeKey: safeKey, + blobId: blobId, + status: err? String(err): 'SUCCESS', + }); + if (err) { + w.abort(); + return void cb(err); + } + })); + }).nThen(function () { + // archive the proof + return void Env.blobStore.archive.proof(safeKey, blobId, function (err) { + Env.Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", { + safeKey: safeKey, + blobId: blobId, + status: err? String(err): 'SUCCESS', + }); + if (err) { + return void cb("E_PROOF_REMOVAL"); + } + cb(); + }); + }); + } + + Metadata.getMetadata(Env, channelId, function (err, metadata) { + if (err) { return void cb(err); } + if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } + if (!Core.isOwner(metadata, unsafeKey)) { + return void cb('INSUFFICIENT_PERMISSIONS'); + } + // temporarily archive the file + return void Env.msgStore.archiveChannel(channelId, function (e) { + Env.Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', { + unsafeKey: unsafeKey, + channelId: channelId, + status: e? String(e): 'SUCCESS', + }); + cb(e); + }); + }); +}; + +Channel.removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { + nThen(function (w) { + Metadata.getMetadata(Env, channelId, w(function (err, metadata) { + if (err) { return void cb(err); } + if (!Core.hasOwners(metadata)) { + w.abort(); + return void cb('E_NO_OWNERS'); + } + if (!Core.isOwner(metadata, unsafeKey)) { + w.abort(); + return void cb("INSUFFICIENT_PERMISSIONS"); + } + // else fall through to the next block + })); + }).nThen(function () { + Env.msgStore.trimChannel(channelId, hash, function (err) { + if (err) { return void cb(err); } + + + // XXX you must also clear the channel's index from historyKeeper cache + }); + }); +}; + +var ARRAY_LINE = /^\[/; + +/* Files can contain metadata but not content + call back with true if the channel log has no content other than metadata + otherwise false +*/ +Channel.isNewChannel = function (Env, channel, cb) { + if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } + if (channel.length !== 32) { return void cb('INVALID_CHAN'); } + + var done = false; + Env.msgStore.getMessages(channel, function (msg) { + if (done) { return; } + try { + if (typeof(msg) === 'string' && ARRAY_LINE.test(msg)) { + done = true; + return void cb(void 0, false); + } + } catch (e) { + Env.WARN('invalid message read from store', e); + } + }, function () { + if (done) { return; } + // no more messages... + cb(void 0, true); + }); +}; + +/* writePrivateMessage + allows users to anonymously send a message to the channel + prevents their netflux-id from being stored in history + and from being broadcast to anyone that might currently be in the channel + + Otherwise behaves the same as sending to a channel +*/ +Channel.writePrivateMessage = function (Env, args, nfwssCtx, cb) { + var channelId = args[0]; + var msg = args[1]; + + // don't bother handling empty messages + if (!msg) { return void cb("INVALID_MESSAGE"); } + + // don't support anything except regular channels + if (!Core.isValidId(channelId) || channelId.length !== 32) { + return void cb("INVALID_CHAN"); + } + + // We expect a modern netflux-websocket-server instance + // if this API isn't here everything will fall apart anyway + if (!(nfwssCtx && nfwssCtx.historyKeeper && typeof(nfwssCtx.historyKeeper.onChannelMessage) === 'function')) { + return void cb("NOT_IMPLEMENTED"); + } + + // historyKeeper expects something with an 'id' attribute + // it will fail unless you provide it, but it doesn't need anything else + var channelStruct = { + id: channelId, + }; + + // construct a message to store and broadcast + var fullMessage = [ + 0, // idk + null, // normally the netflux id, null isn't rejected, and it distinguishes messages written in this way + "MSG", // indicate that this is a MSG + channelId, // channel id + msg // the actual message content. Generally a string + ]; + + // store the message and do everything else that is typically done when going through historyKeeper + nfwssCtx.historyKeeper.onChannelMessage(nfwssCtx, channelStruct, fullMessage); + + // call back with the message and the target channel. + // historyKeeper will take care of broadcasting it if anyone is in the channel + cb(void 0, { + channel: channelId, + message: fullMessage + }); +}; + diff --git a/lib/commands/metadata.js b/lib/commands/metadata.js index 91b08d8cc..3a21aae0b 100644 --- a/lib/commands/metadata.js +++ b/lib/commands/metadata.js @@ -3,8 +3,8 @@ const Data = module.exports; const Meta = require("../metadata"); const BatchRead = require("../batch-read"); -const WriteQueue = require("./write-queue"); -const Core = require("./commands/core"); +const WriteQueue = require("../write-queue"); +const Core = require("./core"); const batchMetadata = BatchRead("GET_METADATA"); Data.getMetadata = function (Env, channel, cb) { diff --git a/lib/commands/quota.js b/lib/commands/quota.js index e7df14364..92b1c3cd0 100644 --- a/lib/commands/quota.js +++ b/lib/commands/quota.js @@ -2,7 +2,7 @@ /* globals Buffer*/ const Quota = module.exports; -const Core = require("./commands/core"); +const Core = require("./core"); const Util = require("../common-util"); const Package = require('../../package.json'); const Https = require("https"); diff --git a/lib/commands/upload.js b/lib/commands/upload.js new file mode 100644 index 000000000..f89cf2904 --- /dev/null +++ b/lib/commands/upload.js @@ -0,0 +1,37 @@ +/*jshint esversion: 6 */ +const Upload = module.exports; +const Util = require("../common-util"); +const Pinning = require("./pin-rpc"); +const nThen = require("nthen"); + +// upload_status +Upload.upload_status = function (Env, safeKey, filesize, _cb) { // FIXME FILES + var cb = Util.once(Util.mkAsync(_cb)); + + // validate that the provided size is actually a positive number + if (typeof(filesize) !== 'number' && + filesize >= 0) { return void cb('E_INVALID_SIZE'); } + + if (filesize >= Env.maxUploadSize) { return cb('TOO_LARGE'); } + + nThen(function (w) { + var abortAndCB = Util.both(w.abort, cb); + Env.blobStore.status(safeKey, w(function (err, inProgress) { + // if there's an error something is weird + if (err) { return void abortAndCB(err); } + + // we cannot upload two things at once + if (inProgress) { return void abortAndCB(void 0, true); } + })); + }).nThen(function () { + // if yuo're here then there are no pending uploads + // check if you have space in your quota to upload something of this size + Pinning.getFreeSpace(Env, safeKey, function (e, free) { + if (e) { return void cb(e); } + if (filesize >= free) { return cb('NOT_ENOUGH_SPACE'); } + cb(void 0, false); + }); + }); +}; + + diff --git a/lib/rpc.js b/lib/rpc.js index 0ddfdb128..1912868ed 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -2,7 +2,6 @@ const nThen = require("nthen"); const Util = require("./common-util"); -const escapeKeyCharacters = Util.escapeKeyCharacters; const mkEvent = Util.mkEvent; const Core = require("./commands/core"); @@ -11,208 +10,13 @@ const Pinning = require("./commands/pin-rpc"); const Quota = require("./commands/quota"); const Block = require("./commands/block"); const Metadata = require("./commands/metadata"); +const Channel = require("./commands/channel"); +const Upload = require("./commands/upload"); var RPC = module.exports; -var Store = require("../storage/file"); -var BlobStore = require("../storage/blob"); - -var Log; - -var WARN = function (e, output) { - if (e && output) { - Log.warn(e, { - output: output, - message: String(e), - stack: new Error(e).stack, - }); - } -}; - -var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { - if (typeof(channelId) !== 'string' || channelId.length !== 32) { - return cb('INVALID_ARGUMENTS'); - } - - Metadata.getMetadata(Env, channelId, function (err, metadata) { - if (err) { return void cb(err); } - if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } - // Confirm that the channel is owned by the user in question - if (!Core.isOwner(metadata, unsafeKey)) { - return void cb('INSUFFICIENT_PERMISSIONS'); - } - return void Env.msgStore.clearChannel(channelId, function (e) { - cb(e); - }); - }); -}; - -var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { - if (typeof(channelId) !== 'string' || !Core.isValidId(channelId)) { - return cb('INVALID_ARGUMENTS'); - } - - if (Env.blobStore.isFileId(channelId)) { - var safeKey = escapeKeyCharacters(unsafeKey); - var blobId = channelId; - - return void nThen(function (w) { - // check if you have permissions - Env.blobStore.isOwnedBy(safeKey, blobId, w(function (err, owned) { - if (err || !owned) { - w.abort(); - return void cb("INSUFFICIENT_PERMISSIONS"); - } - })); - }).nThen(function (w) { - // remove the blob - return void Env.blobStore.archive.blob(blobId, w(function (err) { - Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', { - safeKey: safeKey, - blobId: blobId, - status: err? String(err): 'SUCCESS', - }); - if (err) { - w.abort(); - return void cb(err); - } - })); - }).nThen(function () { - // archive the proof - return void Env.blobStore.archive.proof(safeKey, blobId, function (err) { - Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", { - safeKey: safeKey, - blobId: blobId, - status: err? String(err): 'SUCCESS', - }); - if (err) { - return void cb("E_PROOF_REMOVAL"); - } - cb(); - }); - }); - } - - Metadata.getMetadata(Env, channelId, function (err, metadata) { - if (err) { return void cb(err); } - if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } - if (!Core.isOwner(metadata, unsafeKey)) { - return void cb('INSUFFICIENT_PERMISSIONS'); - } - // temporarily archive the file - return void Env.msgStore.archiveChannel(channelId, function (e) { - Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', { - unsafeKey: unsafeKey, - channelId: channelId, - status: e? String(e): 'SUCCESS', - }); - cb(e); - }); - }); -}; - -var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { - nThen(function (w) { - Metadata.getMetadata(Env, channelId, w(function (err, metadata) { - if (err) { return void cb(err); } - if (!Core.hasOwners(metadata)) { - w.abort(); - return void cb('E_NO_OWNERS'); - } - if (!Core.isOwner(metadata, unsafeKey)) { - w.abort(); - return void cb("INSUFFICIENT_PERMISSIONS"); - } - // else fall through to the next block - })); - }).nThen(function () { - Env.msgStore.trimChannel(channelId, hash, function (err) { - if (err) { return void cb(err); } - - - // XXX you must also clear the channel's index from historyKeeper cache - }); - }); -}; - -var ARRAY_LINE = /^\[/; - -/* Files can contain metadata but not content - call back with true if the channel log has no content other than metadata - otherwise false -*/ -var isNewChannel = function (Env, channel, cb) { - if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } - if (channel.length !== 32) { return void cb('INVALID_CHAN'); } - - var done = false; - Env.msgStore.getMessages(channel, function (msg) { - if (done) { return; } - try { - if (typeof(msg) === 'string' && ARRAY_LINE.test(msg)) { - done = true; - return void cb(void 0, false); - } - } catch (e) { - WARN('invalid message read from store', e); - } - }, function () { - if (done) { return; } - // no more messages... - cb(void 0, true); - }); -}; - -/* writePrivateMessage - allows users to anonymously send a message to the channel - prevents their netflux-id from being stored in history - and from being broadcast to anyone that might currently be in the channel - - Otherwise behaves the same as sending to a channel -*/ -var writePrivateMessage = function (Env, args, nfwssCtx, cb) { - var channelId = args[0]; - var msg = args[1]; - - // don't bother handling empty messages - if (!msg) { return void cb("INVALID_MESSAGE"); } - - // don't support anything except regular channels - if (!Core.isValidId(channelId) || channelId.length !== 32) { - return void cb("INVALID_CHAN"); - } - - // We expect a modern netflux-websocket-server instance - // if this API isn't here everything will fall apart anyway - if (!(nfwssCtx && nfwssCtx.historyKeeper && typeof(nfwssCtx.historyKeeper.onChannelMessage) === 'function')) { - return void cb("NOT_IMPLEMENTED"); - } - - // historyKeeper expects something with an 'id' attribute - // it will fail unless you provide it, but it doesn't need anything else - var channelStruct = { - id: channelId, - }; - - // construct a message to store and broadcast - var fullMessage = [ - 0, // idk - null, // normally the netflux id, null isn't rejected, and it distinguishes messages written in this way - "MSG", // indicate that this is a MSG - channelId, // channel id - msg // the actual message content. Generally a string - ]; - - // store the message and do everything else that is typically done when going through historyKeeper - nfwssCtx.historyKeeper.onChannelMessage(nfwssCtx, channelStruct, fullMessage); - - // call back with the message and the target channel. - // historyKeeper will take care of broadcasting it if anyone is in the channel - cb(void 0, { - channel: channelId, - message: fullMessage - }); -}; +const Store = require("../storage/file"); +const BlobStore = require("../storage/blob"); var isUnauthenticatedCall = function (call) { return [ @@ -254,38 +58,8 @@ var isAuthenticatedCall = function (call) { ].indexOf(call) !== -1; }; -// upload_status -var upload_status = function (Env, safeKey, filesize, _cb) { // FIXME FILES - var cb = Util.once(Util.mkAsync(_cb)); - - // validate that the provided size is actually a positive number - if (typeof(filesize) !== 'number' && - filesize >= 0) { return void cb('E_INVALID_SIZE'); } - - if (filesize >= Env.maxUploadSize) { return cb('TOO_LARGE'); } - - nThen(function (w) { - var abortAndCB = Util.both(w.abort, cb); - Env.blobStore.status(safeKey, w(function (err, inProgress) { - // if there's an error something is weird - if (err) { return void abortAndCB(err); } - - // we cannot upload two things at once - if (inProgress) { return void abortAndCB(void 0, true); } - })); - }).nThen(function () { - // if yuo're here then there are no pending uploads - // check if you have space in your quota to upload something of this size - Pinning.getFreeSpace(Env, safeKey, function (e, free) { - if (e) { return void cb(e); } - if (filesize >= free) { return cb('NOT_ENOUGH_SPACE'); } - cb(void 0, false); - }); - }); -}; - RPC.create = function (config, cb) { - Log = config.log; + var Log = config.log; // load pin-store... Log.silly('LOADING RPC MODULE'); @@ -294,6 +68,16 @@ RPC.create = function (config, cb) { return typeof(config[key]) === 'string'? config[key]: def; }; + var WARN = function (e, output) { + if (e && output) { + Log.warn(e, { + output: output, + message: String(e), + stack: new Error(e).stack, + }); + } + }; + var Env = { defaultStorageLimit: config.defaultStorageLimit, maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024), @@ -382,11 +166,11 @@ RPC.create = function (config, cb) { respond(null, [null, isPinned, null]); }); case 'IS_NEW_CHANNEL': - return void isNewChannel(Env, msg[1], function (e, isNew) { + return void Channel.isNewChannel(Env, msg[1], function (e, isNew) { respond(e, [null, isNew, null]); }); case 'WRITE_PRIVATE_MESSAGE': - return void writePrivateMessage(Env, msg[1], nfwssCtx, function (e, output) { + return void Channel.writePrivateMessage(Env, msg[1], nfwssCtx, function (e, output) { respond(e, output); }); default: @@ -450,7 +234,7 @@ RPC.create = function (config, cb) { return void respond("INVALID_RPC_CALL"); } - var safeKey = escapeKeyCharacters(publicKey); + var safeKey = Util.escapeKeyCharacters(publicKey); /* If you have gotten this far, you have signed the message with the public key which you provided. @@ -528,18 +312,18 @@ RPC.create = function (config, cb) { Respond(void 0, "OK"); }); case 'CLEAR_OWNED_CHANNEL': - return void clearOwnedChannel(Env, msg[1], publicKey, function (e, response) { + return void Channel.clearOwnedChannel(Env, msg[1], publicKey, function (e, response) { if (e) { return void Respond(e); } Respond(void 0, response); }); case 'REMOVE_OWNED_CHANNEL': - return void removeOwnedChannel(Env, msg[1], publicKey, function (e) { + return void Channel.removeOwnedChannel(Env, msg[1], publicKey, function (e) { if (e) { return void Respond(e); } Respond(void 0, "OK"); }); case 'TRIM_OWNED_CHANNEL_HISTORY': - return void removeOwnedChannelHistory(Env, msg[1], publicKey, msg[2], function (e) { + return void Channel.removeOwnedChannelHistory(Env, msg[1], publicKey, msg[2], function (e) { if (e) { return void Respond(e); } Respond(void 0, 'OK'); }); @@ -560,7 +344,7 @@ RPC.create = function (config, cb) { }); case 'UPLOAD_STATUS': var filesize = msg[1]; - return void upload_status(Env, safeKey, filesize, function (e, yes) { + return void Upload.upload_status(Env, safeKey, filesize, function (e, yes) { if (!e && !yes) { // no pending uploads, set the new size var user = Core.getSession(Sessions, safeKey); From b093d3f0d2cb75472e15235e0cbd4589b8d5cbff Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 14:45:53 -0500 Subject: [PATCH 28/53] WIP massive rpc refactor --- lib/commands/admin-rpc.js | 4 +- lib/commands/quota.js | 31 +- lib/rpc.js | 713 ++++++++++++++++++++------------------ 3 files changed, 386 insertions(+), 362 deletions(-) diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 9e32fe2a8..24a8e4348 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -93,7 +93,7 @@ var getDiskUsage = function (Env, cb) { -Admin.command = function (Env, ctx, publicKey, config, data, cb) { +Admin.command = function (Env, ctx, publicKey, data, cb) { var admins = Env.admins; if (admins.indexOf(publicKey) === -1) { return void cb("FORBIDDEN"); @@ -109,7 +109,7 @@ Admin.command = function (Env, ctx, publicKey, config, data, cb) { case 'DISK_USAGE': return getDiskUsage(Env, cb); case 'FLUSH_CACHE': - config.flushCache(); + Env.flushCache(); return cb(void 0, true); case 'SHUTDOWN': return shutdown(Env, ctx, cb); diff --git a/lib/commands/quota.js b/lib/commands/quota.js index 92b1c3cd0..b74195821 100644 --- a/lib/commands/quota.js +++ b/lib/commands/quota.js @@ -7,7 +7,7 @@ const Util = require("../common-util"); const Package = require('../../package.json'); const Https = require("https"); -Quota.applyCustomLimits = function (Env, config) { +Quota.applyCustomLimits = function (Env) { var isLimit = function (o) { var valid = o && typeof(o) === 'object' && typeof(o.limit) === 'number' && @@ -16,7 +16,7 @@ Quota.applyCustomLimits = function (Env, config) { return valid; }; - // read custom limits from the config + // read custom limits from the Environment (taken from config) var customLimits = (function (custom) { var limits = {}; Object.keys(custom).forEach(function (k) { @@ -27,7 +27,7 @@ Quota.applyCustomLimits = function (Env, config) { }); }); return limits; - }(config.customLimits || {})); + }(Env.customLimits || {})); Object.keys(customLimits).forEach(function (k) { if (!isLimit(customLimits[k])) { return; } @@ -37,17 +37,18 @@ Quota.applyCustomLimits = function (Env, config) { // The limits object contains storage limits for all the publicKey that have paid // To each key is associated an object containing the 'limit' value and a 'note' explaining that limit -Quota.updateLimits = function (Env, config, publicKey, cb) { // FIXME BATCH?S +// XXX maybe the use case with a publicKey should be a different command that calls this? +Quota.updateLimits = function (Env, publicKey, cb) { // FIXME BATCH?S - if (config.adminEmail === false) { - Quota.applyCustomLimits(Env, config); - if (config.allowSubscriptions === false) { return; } + if (Env.adminEmail === false) { + Quota.applyCustomLimits(Env); + if (Env.allowSubscriptions === false) { return; } throw new Error("allowSubscriptions must be false if adminEmail is false"); } if (typeof cb !== "function") { cb = function () {}; } - var defaultLimit = typeof(config.defaultStorageLimit) === 'number'? - config.defaultStorageLimit: Core.DEFAULT_LIMIT; + var defaultLimit = typeof(Env.defaultStorageLimit) === 'number'? + Env.defaultStorageLimit: Core.DEFAULT_LIMIT; var userId; if (publicKey) { @@ -55,9 +56,9 @@ Quota.updateLimits = function (Env, config, publicKey, cb) { // FIXME BATCH?S } var body = JSON.stringify({ - domain: config.myDomain, - subdomain: config.mySubdomain || null, - adminEmail: config.adminEmail, + domain: Env.myDomain, + subdomain: Env.mySubdomain || null, + adminEmail: Env.adminEmail, version: Package.version }); var options = { @@ -84,7 +85,7 @@ Quota.updateLimits = function (Env, config, publicKey, cb) { // FIXME BATCH?S try { var json = JSON.parse(str); Env.limits = json; - Quota.applyCustomLimits(Env, config); + Quota.applyCustomLimits(Env); var l; if (userId) { @@ -100,8 +101,8 @@ Quota.updateLimits = function (Env, config, publicKey, cb) { // FIXME BATCH?S }); req.on('error', function (e) { - Quota.applyCustomLimits(Env, config); - if (!config.domain) { return cb(); } + Quota.applyCustomLimits(Env); + if (!Env.domain) { return cb(); } // XXX cb(e); }); diff --git a/lib/rpc.js b/lib/rpc.js index 1912868ed..171fb590f 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -18,44 +18,359 @@ var RPC = module.exports; const Store = require("../storage/file"); const BlobStore = require("../storage/blob"); +const UNAUTHENTICATED_CALLS = [ + 'GET_FILE_SIZE', + 'GET_METADATA', + 'GET_MULTIPLE_FILE_SIZE', + 'IS_CHANNEL_PINNED', + 'IS_NEW_CHANNEL', + 'GET_HISTORY_OFFSET', + 'GET_DELETED_PADS', + 'WRITE_PRIVATE_MESSAGE', +]; + var isUnauthenticatedCall = function (call) { - return [ - 'GET_FILE_SIZE', - 'GET_METADATA', - 'GET_MULTIPLE_FILE_SIZE', - 'IS_CHANNEL_PINNED', - 'IS_NEW_CHANNEL', - 'GET_HISTORY_OFFSET', - 'GET_DELETED_PADS', - 'WRITE_PRIVATE_MESSAGE', - ].indexOf(call) !== -1; + return UNAUTHENTICATED_CALLS.indexOf(call) !== -1; }; +const AUTHENTICATED_CALLS = [ + 'COOKIE', + 'RESET', + 'PIN', + 'UNPIN', + 'GET_HASH', + 'GET_TOTAL_SIZE', + 'UPDATE_LIMITS', + 'GET_LIMIT', + 'UPLOAD_STATUS', + 'UPLOAD_COMPLETE', + 'OWNED_UPLOAD_COMPLETE', + 'UPLOAD_CANCEL', + 'EXPIRE_SESSION', + 'TRIM_OWNED_CHANNEL_HISTORY', + 'CLEAR_OWNED_CHANNEL', + 'REMOVE_OWNED_CHANNEL', + 'REMOVE_PINS', + 'TRIM_PINS', + 'WRITE_LOGIN_BLOCK', + 'REMOVE_LOGIN_BLOCK', + 'ADMIN', + 'SET_METADATA' +]; + var isAuthenticatedCall = function (call) { - return [ - 'COOKIE', - 'RESET', - 'PIN', - 'UNPIN', - 'GET_HASH', - 'GET_TOTAL_SIZE', - 'UPDATE_LIMITS', - 'GET_LIMIT', - 'UPLOAD_STATUS', - 'UPLOAD_COMPLETE', - 'OWNED_UPLOAD_COMPLETE', - 'UPLOAD_CANCEL', - 'EXPIRE_SESSION', - 'TRIM_OWNED_CHANNEL_HISTORY', - 'CLEAR_OWNED_CHANNEL', - 'REMOVE_OWNED_CHANNEL', - 'REMOVE_PINS', - 'TRIM_PINS', - 'WRITE_LOGIN_BLOCK', - 'REMOVE_LOGIN_BLOCK', - 'ADMIN', - 'SET_METADATA' - ].indexOf(call) !== -1; + return AUTHENTICATED_CALLS.indexOf(call) !== -1; +}; + +var isUnauthenticateMessage = function (msg) { + return msg && msg.length === 2 && isUnauthenticatedCall(msg[0]); +}; + +var handleUnauthenticatedMessage = function (Env, msg, respond, nfwssCtx) { + Env.Log.silly('LOG_RPC', msg[0]); + switch (msg[0]) { + case 'GET_HISTORY_OFFSET': { // XXX not actually used anywhere? + if (typeof(msg[1]) !== 'object' || typeof(msg[1].channelName) !== 'string') { + return respond('INVALID_ARG_FORMAT', msg); + } + const msgHash = typeof(msg[1].msgHash) === 'string' ? msg[1].msgHash : undefined; + nfwssCtx.getHistoryOffset(nfwssCtx, msg[1].channelName, msgHash, (e, ret) => { + if (e) { + if (e.code !== 'ENOENT') { + Env.WARN(e.stack, msg); + } + return respond(e.message); + } + respond(e, [null, ret, null]); + }); + break; + } + case 'GET_FILE_SIZE': + return void Pinning.getFileSize(Env, msg[1], function (e, size) { + Env.WARN(e, msg[1]); + respond(e, [null, size, null]); + }); + case 'GET_METADATA': + return void Metadata.getMetadata(Env, msg[1], function (e, data) { + Env.WARN(e, msg[1]); + respond(e, [null, data, null]); + }); + case 'GET_MULTIPLE_FILE_SIZE': // XXX not actually used on the client? + return void Pinning.getMultipleFileSize(Env, msg[1], function (e, dict) { + if (e) { + Env.WARN(e, dict); + return respond(e); + } + respond(e, [null, dict, null]); + }); + case 'GET_DELETED_PADS': + return void Pinning.getDeletedPads(Env, msg[1], function (e, list) { + if (e) { + Env.WARN(e, msg[1]); + return respond(e); + } + respond(e, [null, list, null]); + }); + case 'IS_CHANNEL_PINNED': + return void Pinning.isChannelPinned(Env, msg[1], function (isPinned) { + respond(null, [null, isPinned, null]); + }); + case 'IS_NEW_CHANNEL': + return void Channel.isNewChannel(Env, msg[1], function (e, isNew) { + respond(e, [null, isNew, null]); + }); + case 'WRITE_PRIVATE_MESSAGE': + return void Channel.writePrivateMessage(Env, msg[1], nfwssCtx, function (e, output) { + respond(e, output); + }); + default: + Env.Log.warn("UNSUPPORTED_RPC_CALL", msg); + return respond('UNSUPPORTED_RPC_CALL', msg); + } +}; + +var handleAuthenticatedMessage = function (Env, map) { + var msg = map.msg; + var safeKey = map.safeKey; + var publicKey = map.publicKey; + var Respond = map.Respond; + var ctx = map.ctx; + + Env.Log.silly('LOG_RPC', msg[0]); + switch (msg[0]) { + case 'COOKIE': return void Respond(void 0); + case 'RESET': + return Pinning.resetUserPins(Env, safeKey, msg[1], function (e, hash) { // XXX USER_TARGETED + //WARN(e, hash); + return void Respond(e, hash); + }); + case 'PIN': + return Pinning.pinChannel(Env, safeKey, msg[1], function (e, hash) { // XXX USER_TARGETED + Env.WARN(e, hash); + Respond(e, hash); + }); + case 'UNPIN': + return Pinning.unpinChannel(Env, safeKey, msg[1], function (e, hash) { // XXX USER_TARGETED + Env.WARN(e, hash); + Respond(e, hash); + }); + case 'GET_HASH': + return void Pinning.getHash(Env, safeKey, function (e, hash) { // XXX USER_SCOPED + Env.WARN(e, hash); + Respond(e, hash); + }); + case 'GET_TOTAL_SIZE': // TODO cache this, since it will get called quite a bit + return Pinning.getTotalSize(Env, safeKey, function (e, size) { // XXX USER_SCOPED + if (e) { + Env.WARN(e, safeKey); + return void Respond(e); + } + Respond(e, size); + }); + case 'UPDATE_LIMITS': + return void Quota.updateLimits(Env, safeKey, function (e, limit) { // XXX USER_SCOPED + if (e) { + Env.WARN(e, limit); + return void Respond(e); + } + Respond(void 0, limit); + }); + case 'GET_LIMIT': + return void Pinning.getLimit(Env, safeKey, function (e, limit) { // XXX USER_SCOPED + if (e) { + Env.WARN(e, limit); + return void Respond(e); + } + Respond(void 0, limit); + }); + case 'EXPIRE_SESSION': + return void setTimeout(function () { // XXX USER_SCOPED + Core.expireSession(Env.Sessions, safeKey); + Respond(void 0, "OK"); + }); + case 'CLEAR_OWNED_CHANNEL': + return void Channel.clearOwnedChannel(Env, msg[1], publicKey, function (e, response) { // XXX USER_TARGETD_INVERSE + if (e) { return void Respond(e); } + Respond(void 0, response); + }); + + case 'REMOVE_OWNED_CHANNEL': + return void Channel.removeOwnedChannel(Env, msg[1], publicKey, function (e) { // XXX USER_TARGETED_INVERSE + if (e) { return void Respond(e); } + Respond(void 0, "OK"); + }); + case 'TRIM_OWNED_CHANNEL_HISTORY': + return void Channel.removeOwnedChannelHistory(Env, msg[1], publicKey, msg[2], function (e) { // XXX USER_TARGETED_DOUBLE + if (e) { return void Respond(e); } + Respond(void 0, 'OK'); + }); + case 'REMOVE_PINS': + return void Pinning.removePins(Env, safeKey, function (e) { // XXX USER_SCOPED + if (e) { return void Respond(e); } + Respond(void 0, "OK"); + }); + case 'TRIM_PINS': + return void Pinning.trimPins(Env, safeKey, function (e) { // XXX USER_SCOPED + if (e) { return void Respond(e); } + Respond(void 0, "OK"); + }); + case 'UPLOAD': + return void Env.blobStore.upload(safeKey, msg[1], function (e, len) { // XXX USER_SCOPED_SPECIAL + Env.WARN(e, len); + Respond(e, len); + }); + case 'UPLOAD_STATUS': + var filesize = msg[1]; + return void Upload.upload_status(Env, safeKey, filesize, function (e, yes) { // XXX USER_TARGETED + if (!e && !yes) { + // no pending uploads, set the new size + var user = Core.getSession(Env.Sessions, safeKey); + user.pendingUploadSize = filesize; + user.currentUploadSize = 0; + } + Respond(e, yes); + }); + case 'UPLOAD_COMPLETE': + return void Env.blobStore.complete(safeKey, msg[1], function (e, hash) { // XXX USER_SCOPED_SPECIAL + Env.WARN(e, hash); + Respond(e, hash); + }); + case 'OWNED_UPLOAD_COMPLETE': + return void Env.blobStore.completeOwned(safeKey, msg[1], function (e, blobId) { // XXX USER_SCOPED_SPECIAL + Env.WARN(e, blobId); + Respond(e, blobId); + }); + case 'UPLOAD_CANCEL': + // msg[1] is fileSize + // if we pass it here, we can start an upload right away without calling + // UPLOAD_STATUS again + return void Env.blobStore.cancel(safeKey, msg[1], function (e) { // XXX USER_SCOPED_SPECIAL + Env.WARN(e, 'UPLOAD_CANCEL'); + Respond(e); + }); + case 'WRITE_LOGIN_BLOCK': + return void Block.writeLoginBlock(Env, msg[1], function (e) { // XXX SPECIAL + if (e) { + Env.WARN(e, 'WRITE_LOGIN_BLOCK'); + return void Respond(e); + } + Respond(e); + }); + case 'REMOVE_LOGIN_BLOCK': + return void Block.removeLoginBlock(Env, msg[1], function (e) { // XXX SPECIAL + if (e) { + Env.WARN(e, 'REMOVE_LOGIN_BLOCK'); + return void Respond(e); + } + Respond(e); + }); + case 'ADMIN': + return void Admin.command(Env, ctx, safeKey, msg[1], function (e, result) { // XXX SPECIAL + if (e) { + Env.WARN(e, result); + return void Respond(e); + } + Respond(void 0, result); + }); + case 'SET_METADATA': + return void Metadata.setMetadata(Env, msg[1], publicKey, function (e, data) { // XXX USER_TARGETED_INVERSE + if (e) { + Env.WARN(e, data); + return void Respond(e); + } + Respond(void 0, data); + }); + default: + return void Respond('UNSUPPORTED_RPC_CALL', msg); + } +}; + +var rpc = function (Env, ctx, data, respond) { + if (!Array.isArray(data)) { + Env.Log.debug('INVALID_ARG_FORMET', data); + return void respond('INVALID_ARG_FORMAT'); + } + + if (!data.length) { + return void respond("INSUFFICIENT_ARGS"); + } else if (data.length !== 1) { + Env.Log.debug('UNEXPECTED_ARGUMENTS_LENGTH', data); + } + + var msg = data[0].slice(0); + + if (!Array.isArray(msg)) { + return void respond('INVALID_ARG_FORMAT'); + } + + if (isUnauthenticateMessage(msg)) { + return handleUnauthenticatedMessage(Env, msg, respond, ctx); + } + + var signature = msg.shift(); + var publicKey = msg.shift(); + + // make sure a user object is initialized in the cookie jar + if (publicKey) { + Core.getSession(Env.Sessions, publicKey); + } else { + Env.Log.debug("NO_PUBLIC_KEY_PROVIDED", publicKey); + } + + var cookie = msg[0]; + if (!Core.isValidCookie(Env.Sessions, publicKey, cookie)) { + // no cookie is fine if the RPC is to get a cookie + if (msg[1] !== 'COOKIE') { + return void respond('NO_COOKIE'); + } + } + + var serialized = JSON.stringify(msg); + + if (!(serialized && typeof(publicKey) === 'string')) { + return void respond('INVALID_MESSAGE_OR_PUBLIC_KEY'); + } + + if (isAuthenticatedCall(msg[1])) { + if (Core.checkSignature(Env, serialized, signature, publicKey) !== true) { + return void respond("INVALID_SIGNATURE_OR_PUBLIC_KEY"); + } + } else if (msg[1] !== 'UPLOAD') { + Env.Log.warn('INVALID_RPC_CALL', msg[1]); + return void respond("INVALID_RPC_CALL"); + } + + var safeKey = Util.escapeKeyCharacters(publicKey); + /* If you have gotten this far, you have signed the message with the + public key which you provided. + + We can safely modify the state for that key + + OR it's an unauthenticated call, which must not modify the state + for that key in a meaningful way. + */ + + // discard validated cookie from message + msg.shift(); + + var Respond = function (e, msg) { + var session = Env.Sessions[safeKey]; + var token = session? session.tokens.slice(-1)[0]: ''; + var cookie = Core.makeCookie(token).join('|'); + respond(e ? String(e): e, [cookie].concat(typeof(msg) !== 'undefined' ?msg: [])); + }; + + if (typeof(msg) !== 'object' || !msg.length) { + return void Respond('INVALID_MSG'); + } + + handleAuthenticatedMessage(Env, { + msg: msg, + safeKey: safeKey, + publicKey: publicKey, + Respond: Respond, + ctx: ctx, + }); }; RPC.create = function (config, cb) { @@ -92,6 +407,13 @@ RPC.create = function (config, cb) { sessionExpirationInterval: undefined, Log: Log, WARN: WARN, + flushCache: config.flushCache, + adminEmail: config.adminEmail, + allowSubscriptions: config.allowSubscriptions, + myDomain: config.myDomain, + mySubdomain: config.mySubdomain, + customLimits: config.customLimits, + domain: config.domain // XXX }; try { @@ -112,322 +434,14 @@ RPC.create = function (config, cb) { paths.staging = keyOrDefaultString('blobStagingPath', './blobstage'); paths.blob = keyOrDefaultString('blobPath', './blob'); - var isUnauthenticateMessage = function (msg) { - return msg && msg.length === 2 && isUnauthenticatedCall(msg[0]); - }; - - var handleUnauthenticatedMessage = function (msg, respond, nfwssCtx) { - Log.silly('LOG_RPC', msg[0]); - switch (msg[0]) { - case 'GET_HISTORY_OFFSET': { - if (typeof(msg[1]) !== 'object' || typeof(msg[1].channelName) !== 'string') { - return respond('INVALID_ARG_FORMAT', msg); - } - const msgHash = typeof(msg[1].msgHash) === 'string' ? msg[1].msgHash : undefined; - nfwssCtx.getHistoryOffset(nfwssCtx, msg[1].channelName, msgHash, (e, ret) => { - if (e) { - if (e.code !== 'ENOENT') { - WARN(e.stack, msg); - } - return respond(e.message); - } - respond(e, [null, ret, null]); - }); - break; - } - case 'GET_FILE_SIZE': - return void Pinning.getFileSize(Env, msg[1], function (e, size) { - WARN(e, msg[1]); - respond(e, [null, size, null]); - }); - case 'GET_METADATA': - return void Metadata.getMetadata(Env, msg[1], function (e, data) { - WARN(e, msg[1]); - respond(e, [null, data, null]); - }); - case 'GET_MULTIPLE_FILE_SIZE': - return void Pinning.getMultipleFileSize(Env, msg[1], function (e, dict) { - if (e) { - WARN(e, dict); - return respond(e); - } - respond(e, [null, dict, null]); - }); - case 'GET_DELETED_PADS': - return void Pinning.getDeletedPads(Env, msg[1], function (e, list) { - if (e) { - WARN(e, msg[1]); - return respond(e); - } - respond(e, [null, list, null]); - }); - case 'IS_CHANNEL_PINNED': - return void Pinning.isChannelPinned(Env, msg[1], function (isPinned) { - respond(null, [null, isPinned, null]); - }); - case 'IS_NEW_CHANNEL': - return void Channel.isNewChannel(Env, msg[1], function (e, isNew) { - respond(e, [null, isNew, null]); - }); - case 'WRITE_PRIVATE_MESSAGE': - return void Channel.writePrivateMessage(Env, msg[1], nfwssCtx, function (e, output) { - respond(e, output); - }); - default: - Log.warn("UNSUPPORTED_RPC_CALL", msg); - return respond('UNSUPPORTED_RPC_CALL', msg); - } - }; - - var rpc0 = function (ctx, data, respond) { - if (!Array.isArray(data)) { - Log.debug('INVALID_ARG_FORMET', data); - return void respond('INVALID_ARG_FORMAT'); - } - - if (!data.length) { - return void respond("INSUFFICIENT_ARGS"); - } else if (data.length !== 1) { - Log.debug('UNEXPECTED_ARGUMENTS_LENGTH', data); - } - - var msg = data[0].slice(0); - - if (!Array.isArray(msg)) { - return void respond('INVALID_ARG_FORMAT'); - } - - if (isUnauthenticateMessage(msg)) { - return handleUnauthenticatedMessage(msg, respond, ctx); - } - - var signature = msg.shift(); - var publicKey = msg.shift(); - - // make sure a user object is initialized in the cookie jar - if (publicKey) { - Core.getSession(Sessions, publicKey); - } else { - Log.debug("NO_PUBLIC_KEY_PROVIDED", publicKey); - } - - var cookie = msg[0]; - if (!Core.isValidCookie(Sessions, publicKey, cookie)) { - // no cookie is fine if the RPC is to get a cookie - if (msg[1] !== 'COOKIE') { - return void respond('NO_COOKIE'); - } - } - - var serialized = JSON.stringify(msg); - - if (!(serialized && typeof(publicKey) === 'string')) { - return void respond('INVALID_MESSAGE_OR_PUBLIC_KEY'); - } - - if (isAuthenticatedCall(msg[1])) { - if (Core.checkSignature(Env, serialized, signature, publicKey) !== true) { - return void respond("INVALID_SIGNATURE_OR_PUBLIC_KEY"); - } - } else if (msg[1] !== 'UPLOAD') { - Log.warn('INVALID_RPC_CALL', msg[1]); - return void respond("INVALID_RPC_CALL"); - } - - var safeKey = Util.escapeKeyCharacters(publicKey); - /* If you have gotten this far, you have signed the message with the - public key which you provided. - - We can safely modify the state for that key - - OR it's an unauthenticated call, which must not modify the state - for that key in a meaningful way. - */ - - // discard validated cookie from message - msg.shift(); - - var Respond = function (e, msg) { - var session = Sessions[safeKey]; - var token = session? session.tokens.slice(-1)[0]: ''; - var cookie = Core.makeCookie(token).join('|'); - respond(e ? String(e): e, [cookie].concat(typeof(msg) !== 'undefined' ?msg: [])); - }; - - if (typeof(msg) !== 'object' || !msg.length) { - return void Respond('INVALID_MSG'); - } - - var handleMessage = function () { - Log.silly('LOG_RPC', msg[0]); - switch (msg[0]) { - case 'COOKIE': return void Respond(void 0); - case 'RESET': - return Pinning.resetUserPins(Env, safeKey, msg[1], function (e, hash) { - //WARN(e, hash); - return void Respond(e, hash); - }); - case 'PIN': - return Pinning.pinChannel(Env, safeKey, msg[1], function (e, hash) { - WARN(e, hash); - Respond(e, hash); - }); - case 'UNPIN': - return Pinning.unpinChannel(Env, safeKey, msg[1], function (e, hash) { - WARN(e, hash); - Respond(e, hash); - }); - case 'GET_HASH': - return void Pinning.getHash(Env, safeKey, function (e, hash) { - WARN(e, hash); - Respond(e, hash); - }); - case 'GET_TOTAL_SIZE': // TODO cache this, since it will get called quite a bit - return Pinning.getTotalSize(Env, safeKey, function (e, size) { - if (e) { - WARN(e, safeKey); - return void Respond(e); - } - Respond(e, size); - }); - case 'UPDATE_LIMITS': - return void Quota.updateLimits(Env, config, safeKey, function (e, limit) { - if (e) { - WARN(e, limit); - return void Respond(e); - } - Respond(void 0, limit); - }); - case 'GET_LIMIT': - return void Pinning.getLimit(Env, safeKey, function (e, limit) { - if (e) { - WARN(e, limit); - return void Respond(e); - } - Respond(void 0, limit); - }); - case 'EXPIRE_SESSION': - return void setTimeout(function () { - Core.expireSession(Sessions, safeKey); - Respond(void 0, "OK"); - }); - case 'CLEAR_OWNED_CHANNEL': - return void Channel.clearOwnedChannel(Env, msg[1], publicKey, function (e, response) { - if (e) { return void Respond(e); } - Respond(void 0, response); - }); - - case 'REMOVE_OWNED_CHANNEL': - return void Channel.removeOwnedChannel(Env, msg[1], publicKey, function (e) { - if (e) { return void Respond(e); } - Respond(void 0, "OK"); - }); - case 'TRIM_OWNED_CHANNEL_HISTORY': - return void Channel.removeOwnedChannelHistory(Env, msg[1], publicKey, msg[2], function (e) { - if (e) { return void Respond(e); } - Respond(void 0, 'OK'); - }); - case 'REMOVE_PINS': - return void Pinning.removePins(Env, safeKey, function (e) { - if (e) { return void Respond(e); } - Respond(void 0, "OK"); - }); - case 'TRIM_PINS': - return void Pinning.trimPins(Env, safeKey, function (e) { - if (e) { return void Respond(e); } - Respond(void 0, "OK"); - }); - case 'UPLOAD': - return void Env.blobStore.upload(safeKey, msg[1], function (e, len) { - WARN(e, len); - Respond(e, len); - }); - case 'UPLOAD_STATUS': - var filesize = msg[1]; - return void Upload.upload_status(Env, safeKey, filesize, function (e, yes) { - if (!e && !yes) { - // no pending uploads, set the new size - var user = Core.getSession(Sessions, safeKey); - user.pendingUploadSize = filesize; - user.currentUploadSize = 0; - } - Respond(e, yes); - }); - case 'UPLOAD_COMPLETE': - return void Env.blobStore.complete(safeKey, msg[1], function (e, hash) { - WARN(e, hash); - Respond(e, hash); - }); - case 'OWNED_UPLOAD_COMPLETE': - return void Env.blobStore.completeOwned(safeKey, msg[1], function (e, blobId) { - WARN(e, blobId); - Respond(e, blobId); - }); - case 'UPLOAD_CANCEL': - // msg[1] is fileSize - // if we pass it here, we can start an upload right away without calling - // UPLOAD_STATUS again - return void Env.blobStore.cancel(safeKey, msg[1], function (e) { - WARN(e, 'UPLOAD_CANCEL'); - Respond(e); - }); - case 'WRITE_LOGIN_BLOCK': - return void Block.writeLoginBlock(Env, msg[1], function (e) { - if (e) { - WARN(e, 'WRITE_LOGIN_BLOCK'); - return void Respond(e); - } - Respond(e); - }); - case 'REMOVE_LOGIN_BLOCK': - return void Block.removeLoginBlock(Env, msg[1], function (e) { - if (e) { - WARN(e, 'REMOVE_LOGIN_BLOCK'); - return void Respond(e); - } - Respond(e); - }); - case 'ADMIN': - return void Admin.command(Env, ctx, safeKey, config, msg[1], function (e, result) { - if (e) { - WARN(e, result); - return void Respond(e); - } - Respond(void 0, result); - }); - case 'SET_METADATA': - return void Metadata.setMetadata(Env, msg[1], publicKey, function (e, data) { - if (e) { - WARN(e, data); - return void Respond(e); - } - Respond(void 0, data); - }); - default: - return void Respond('UNSUPPORTED_RPC_CALL', msg); - } - }; - - handleMessage(true); - }; - - var rpc = function (ctx, data, respond) { - try { - return rpc0(ctx, data, respond); - } catch (e) { - console.log("Error from RPC with data " + JSON.stringify(data)); - console.log(e.stack); - } - }; - var updateLimitDaily = function () { - Quota.updateLimits(Env, config, undefined, function (e) { + Quota.updateLimits(Env, undefined, function (e) { if (e) { WARN('limitUpdate', e); } }); }; - Quota.applyCustomLimits(Env, config); + Quota.applyCustomLimits(Env); updateLimitDaily(); setInterval(updateLimitDaily, 24*3600*1000); @@ -451,7 +465,16 @@ RPC.create = function (config, cb) { Env.blobStore = blob; })); }).nThen(function () { - cb(void 0, rpc); + // XXX it's ugly that we pass ctx and Env separately + // when they're effectively the same thing... + cb(void 0, function (ctx, data, respond) { + try { + return rpc(Env, ctx, data, respond); + } catch (e) { + console.log("Error from RPC with data " + JSON.stringify(data)); + console.log(e.stack); + } + }); // expire old sessions once per minute // XXX allow for graceful shutdown Env.sessionExpirationInterval = setInterval(function () { From 80c012f34d018b32eebe21bd03cc721924840cad Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 27 Jan 2020 17:57:39 -0500 Subject: [PATCH 29/53] prepare to merge history keeper and rpc --- lib/api.js | 60 +++++++++++++++++++++++ lib/historyKeeper.js | 7 +-- server.js | 114 +++++++++++-------------------------------- 3 files changed, 91 insertions(+), 90 deletions(-) create mode 100644 lib/api.js diff --git a/lib/api.js b/lib/api.js new file mode 100644 index 000000000..60296d9b5 --- /dev/null +++ b/lib/api.js @@ -0,0 +1,60 @@ +/* jshint esversion: 6 */ +const nThen = require("nthen"); +const WebSocketServer = require('ws').Server; +const NetfluxSrv = require('chainpad-server/NetfluxWebsocketSrv'); + +module.exports.create = function (config) { + var historyKeeper; + var rpc; + const log = config.log; + const wsConfig = { + server: config.httpServer, + }; + + nThen(function (w) { + require('../storage/file').create(config, w(function (_store) { + config.store = _store; + })); + }).nThen(function (w) { + require("../storage/tasks").create(config, w(function (e, tasks) { + if (e) { + throw e; + } + config.tasks = tasks; + if (config.disableIntegratedTasks) { return; } + + // XXX support stopping this interval + setInterval(function () { + tasks.runAll(function (err) { + if (err) { + // either TASK_CONCURRENCY or an error with tasks.list + // in either case it is already logged. + } + }); + }, 1000 * 60 * 5); // run every five minutes + })); + }).nThen(function (w) { + require("./rpc").create(config, w(function (e, _rpc) { + if (e) { + w.abort(); + throw e; + } + rpc = _rpc; + })); + }).nThen(function () { + var HK = require('./historyKeeper.js'); + var hkConfig = { + tasks: config.tasks, + rpc: rpc, + store: config.store, + log: log, + }; + // XXX historyKeeper exports a `setConfig` method + historyKeeper = HK.create(hkConfig); + }).nThen(function () { + var wsSrv = new WebSocketServer(wsConfig); + // XXX NetfluxSrv shares some internal functions with historyKeeper + // by passing them to setConfig + NetfluxSrv.run(wsSrv, config, historyKeeper); + }); +}; diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index c3ed67cb7..55d6b1780 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -1,6 +1,5 @@ /* jshint esversion: 6 */ /* global Buffer */ -;(function () { 'use strict'; const nThen = require('nthen'); const Nacl = require('tweetnacl/nacl-fast'); @@ -63,6 +62,8 @@ const isValidValidateKeyString = function (key) { } }; +var CHECKPOINT_PATTERN = /^cp\|(([A-Za-z0-9+\/=]+)\|)?/; + module.exports.create = function (cfg) { const rpc = cfg.rpc; const tasks = cfg.tasks; @@ -385,8 +386,6 @@ module.exports.create = function (cfg) { return true; }; - var CHECKPOINT_PATTERN = /^cp\|(([A-Za-z0-9+\/=]+)\|)?/; - /* onChannelMessage Determine what we should store when a message a broadcasted to a channel" @@ -992,5 +991,3 @@ module.exports.create = function (cfg) { onDirectMessage: onDirectMessage, }; }; - -}()); diff --git a/server.js b/server.js index d2bdcabd8..70479d7ee 100644 --- a/server.js +++ b/server.js @@ -4,17 +4,12 @@ var Express = require('express'); var Http = require('http'); var Fs = require('fs'); -var WebSocketServer = require('ws').Server; -var NetfluxSrv = require('chainpad-server/NetfluxWebsocketSrv'); var Package = require('./package.json'); var Path = require("path"); var nThen = require("nthen"); var config = require("./lib/load-config"); -// support multiple storage back ends -var Storage = require('./storage/file'); - var app = Express(); // mode can be FRESH (default), DEV, or PACKAGE @@ -69,11 +64,9 @@ var setHeaders = (function () { if (Object.keys(headers).length) { return function (req, res) { const h = [ - /^\/pad(2)?\/inner\.html.*/, + /^\/pad\/inner\.html.*/, /^\/common\/onlyoffice\/.*\/index\.html.*/, - /^\/sheet\/inner\.html.*/, - /^\/ooslide\/inner\.html.*/, - /^\/oodoc\/inner\.html.*/, + /^\/(sheet|ooslide|oodoc)\/inner\.html.*/, ].some((regex) => { return regex.test(req.url) }) ? padHeaders : headers; @@ -117,11 +110,6 @@ app.use(function (req, res, next) { app.use(Express.static(__dirname + '/www')); -Fs.exists(__dirname + "/customize", function (e) { - if (e) { return; } - console.log("Cryptpad is customizable, see customize.dist/readme.md for details"); -}); - // FIXME I think this is a regression caused by a recent PR // correct this hack without breaking the contributor's intended behaviour. @@ -207,80 +195,36 @@ app.use(function (req, res, next) { var httpServer = Http.createServer(app); -httpServer.listen(config.httpPort,config.httpAddress,function(){ - var host = config.httpAddress; - var hostName = !host.indexOf(':') ? '[' + host + ']' : host; - - var port = config.httpPort; - var ps = port === 80? '': ':' + port; - - console.log('[%s] server available http://%s%s', new Date().toISOString(), hostName, ps); -}); -if (config.httpSafePort) { - Http.createServer(app).listen(config.httpSafePort, config.httpAddress); -} - -var wsConfig = { server: httpServer }; +nThen(function (w) { + Fs.exists(__dirname + "/customize", w(function (e) { + if (e) { return; } + console.log("Cryptpad is customizable, see customize.dist/readme.md for details"); + })); +}).nThen(function (w) { + httpServer.listen(config.httpPort,config.httpAddress,function(){ + var host = config.httpAddress; + var hostName = !host.indexOf(':') ? '[' + host + ']' : host; -var rpc; -var historyKeeper; + var port = config.httpPort; + var ps = port === 80? '': ':' + port; -var log; + console.log('[%s] server available http://%s%s', new Date().toISOString(), hostName, ps); + }); -// Initialize logging, the the store, then tasks, then rpc, then history keeper and then start the server -var nt = nThen(function (w) { - // set up logger - var Logger = require("./lib/log"); - //console.log("Loading logging module"); - Logger.create(config, w(function (_log) { - log = config.log = _log; - })); -}).nThen(function (w) { - if (config.externalWebsocketURL) { - // if you plan to use an external websocket server - // then you don't need to load any API services other than the logger. - // Just abort. - w.abort(); - return; + if (config.httpSafePort) { + Http.createServer(app).listen(config.httpSafePort, config.httpAddress, w()); } - Storage.create(config, w(function (_store) { - config.store = _store; - })); -}).nThen(function (w) { - var Tasks = require("./storage/tasks"); - Tasks.create(config, w(function (e, tasks) { - if (e) { - throw e; - } - config.tasks = tasks; - if (config.disableIntegratedTasks) { return; } - setInterval(function () { - tasks.runAll(function (err) { - if (err) { - // either TASK_CONCURRENCY or an error with tasks.list - // in either case it is already logged. - } - }); - }, 1000 * 60 * 5); // run every five minutes - })); -}).nThen(function (w) { - require("./lib/rpc").create(config, w(function (e, _rpc) { - if (e) { - w.abort(); - throw e; - } - rpc = _rpc; - })); }).nThen(function () { - var HK = require('./lib/historyKeeper.js'); - var hkConfig = { - tasks: config.tasks, - rpc: rpc, - store: config.store, - log: log, - }; - historyKeeper = HK.create(hkConfig); -}).nThen(function () { - var wsSrv = new WebSocketServer(wsConfig); - NetfluxSrv.run(wsSrv, config, historyKeeper); + var wsConfig = { server: httpServer }; + + // Initialize logging then start the API server + require("./lib/log").create(config, function (_log) { + config.log = _log; + config.httpServer = httpServer; + + if (config.externalWebsocketURL) { return; } + require("./lib/api").create(config); + }); }); + + From b922860339e92f5753edbfba428228f692261914 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 27 Jan 2020 18:54:16 -0500 Subject: [PATCH 30/53] drop usage of historyKeeper.setConfig --- lib/api.js | 7 ++--- lib/historyKeeper.js | 69 ++++++++++++++++++++++++-------------------- lib/hk-util.js | 9 ++++++ 3 files changed, 49 insertions(+), 36 deletions(-) diff --git a/lib/api.js b/lib/api.js index 60296d9b5..d633db59a 100644 --- a/lib/api.js +++ b/lib/api.js @@ -4,7 +4,6 @@ const WebSocketServer = require('ws').Server; const NetfluxSrv = require('chainpad-server/NetfluxWebsocketSrv'); module.exports.create = function (config) { - var historyKeeper; var rpc; const log = config.log; const wsConfig = { @@ -50,11 +49,9 @@ module.exports.create = function (config) { log: log, }; // XXX historyKeeper exports a `setConfig` method - historyKeeper = HK.create(hkConfig); - }).nThen(function () { + var wsSrv = new WebSocketServer(wsConfig); - // XXX NetfluxSrv shares some internal functions with historyKeeper - // by passing them to setConfig + var historyKeeper = HK.create(hkConfig); NetfluxSrv.run(wsSrv, config, historyKeeper); }); }; diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 55d6b1780..f8d3c8a15 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -10,6 +10,8 @@ const WriteQueue = require("./write-queue"); const BatchRead = require("./batch-read"); const Extras = require("./hk-util.js"); +const STANDARD_CHANNEL_LENGTH = Extras.STANDARD_CHANNEL_LENGTH; +const EPHEMERAL_CHANNEL_LENGTH = Extras.EPHEMERAL_CHANNEL_LENGTH; let Log; const now = function () { return (new Date()).getTime(); }; @@ -77,14 +79,6 @@ module.exports.create = function (cfg) { Log.verbose('HK_ID', 'History keeper ID: ' + HISTORY_KEEPER_ID); - let sendMsg = function () {}; - let STANDARD_CHANNEL_LENGTH, EPHEMERAL_CHANNEL_LENGTH; - const setConfig = function (config) { - STANDARD_CHANNEL_LENGTH = config.STANDARD_CHANNEL_LENGTH; - EPHEMERAL_CHANNEL_LENGTH = config.EPHEMERAL_CHANNEL_LENGTH; - sendMsg = config.sendMsg; - }; - /* computeIndex can call back with an error or a computed index which includes: * cpIndex: @@ -326,7 +320,7 @@ module.exports.create = function (cfg) { const historyKeeperBroadcast = function (ctx, channel, msg) { let chan = ctx.channels[channel] || (([] /*:any*/) /*:Chan_t*/); chan.forEach(function (user) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)]); }); }; @@ -707,7 +701,7 @@ module.exports.create = function (cfg) { // parsed[1] is the channel id // parsed[2] is a validation key or an object containing metadata (optionnal) // parsed[3] is the last known hash (optionnal) - sendMsg(ctx, user, [seq, 'ACK']); + ctx.sendMsg(ctx, user, [seq, 'ACK']); var channelName = parsed[1]; var config = parsed[2]; var metadata = {}; @@ -758,7 +752,7 @@ module.exports.create = function (cfg) { // FIXME this is hard to read because 'checkExpired' has side effects if (checkExpired(ctx, channelName)) { return void waitFor.abort(); } // always send metadata with GET_HISTORY requests - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); })); }).nThen(() => { let msgCount = 0; @@ -769,12 +763,12 @@ module.exports.create = function (cfg) { msgCount++; // avoid sending the metadata message a second time if (isMetadataMessage(msg) && metadata_cache[channelName]) { return readMore(); } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)], readMore); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)], readMore); }, (err) => { if (err && err.code !== 'ENOENT') { if (err.message !== 'EINVAL') { Log.error("HK_GET_HISTORY", err); } const parsedMsg = {error:err.message, channel: channelName}; - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); return; } @@ -817,12 +811,12 @@ module.exports.create = function (cfg) { } }); } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(metadata)]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(metadata)]); } // End of history message: let parsedMsg = {state: 1, channel: channelName}; - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); }); }); }; @@ -831,7 +825,7 @@ module.exports.create = function (cfg) { var channelName = parsed[1]; var map = parsed[2]; if (!(map && typeof(map) === 'object')) { - return void sendMsg(ctx, user, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]); + return void ctx.sendMsg(ctx, user, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]); } var oldestKnownHash = map.from; @@ -839,14 +833,14 @@ module.exports.create = function (cfg) { var desiredCheckpoint = map.cpCount; var txid = map.txid; if (typeof(desiredMessages) !== 'number' && typeof(desiredCheckpoint) !== 'number') { - return void sendMsg(ctx, user, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]); + return void ctx.sendMsg(ctx, user, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]); } if (!txid) { - return void sendMsg(ctx, user, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]); + return void ctx.sendMsg(ctx, user, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]); } - sendMsg(ctx, user, [seq, 'ACK']); + ctx.sendMsg(ctx, user, [seq, 'ACK']); return void getOlderHistory(channelName, oldestKnownHash, function (messages) { var toSend = []; if (typeof (desiredMessages) === "number") { @@ -862,11 +856,11 @@ module.exports.create = function (cfg) { } } toSend.forEach(function (msg) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['HISTORY_RANGE', txid, msg])]); }); - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['HISTORY_RANGE_END', txid, channelName]) ]); }); @@ -876,20 +870,20 @@ module.exports.create = function (cfg) { // parsed[1] is the channel id // parsed[2] is a validation key (optionnal) // parsed[3] is the last known hash (optionnal) - sendMsg(ctx, user, [seq, 'ACK']); + ctx.sendMsg(ctx, user, [seq, 'ACK']); // FIXME should we send metadata here too? // none of the clientside code which uses this API needs metadata, but it won't hurt to send it (2019-08-22) return void getHistoryAsync(ctx, parsed[1], -1, false, (msg, readMore) => { if (!msg) { return; } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['FULL_HISTORY', msg])], readMore); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['FULL_HISTORY', msg])], readMore); }, (err) => { let parsedMsg = ['FULL_HISTORY_END', parsed[1]]; if (err) { Log.error('HK_GET_FULL_HISTORY', err.stack); parsedMsg = ['ERROR', parsed[1], err.message]; } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); }); }; @@ -899,12 +893,12 @@ module.exports.create = function (cfg) { /* RPC Calls... */ var rpc_call = parsed.slice(1); - sendMsg(ctx, user, [seq, 'ACK']); + ctx.sendMsg(ctx, user, [seq, 'ACK']); try { // slice off the sequence number and pass in the rest of the message rpc(ctx, rpc_call, function (err, output) { if (err) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', err])]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', err])]); return; } var msg = rpc_call[0].slice(); @@ -930,7 +924,7 @@ module.exports.create = function (cfg) { let chan = ctx.channels[output.channel]; if (chan && chan.length) { chan.forEach(function (user) { - sendMsg(ctx, user, output.message); + ctx.sendMsg(ctx, user, output.message); //[0, null, 'MSG', user.id, JSON.stringify(output.message)]); }); } @@ -940,10 +934,10 @@ module.exports.create = function (cfg) { } // finally, send a response to the client that sent the RPC - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0]].concat(output))]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0]].concat(output))]); }); } catch (e) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]); } }; @@ -982,12 +976,25 @@ module.exports.create = function (cfg) { command(ctx, seq, user, parsed); }; + // XXX every one of these values is exported because + // netfluxWebsocketServer needs them to do some magic historyKeeper things + // we could have netflux emit events and let historyKeeper handle them instead return { id: HISTORY_KEEPER_ID, - setConfig: setConfig, - onChannelMessage: onChannelMessage, + + // XXX dropChannel allows netflux to clear historyKeeper's cache + // maybe it should emit a 'channel_dropped' event instead + // and let historyKeeper decide what to do dropChannel: dropChannel, + + // XXX we don't need to export checkExpired if netflux allows it to be HK's responsibility checkExpired: checkExpired, + + // XXX again, if netflux emitted events then historyKeeper could handle them itself + // and netflux wouldn't need to have historyKeeper-specific code onDirectMessage: onDirectMessage, + + // XXX same + onChannelMessage: onChannelMessage, }; }; diff --git a/lib/hk-util.js b/lib/hk-util.js index cea39c4e1..aaa861054 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -22,3 +22,12 @@ HK.getHash = function (msg, Log) { return msg.slice(0,64); }; +// historyKeeper should explicitly store any channel +// with a 32 character id +HK.STANDARD_CHANNEL_LENGTH = 32; + +// historyKeeper should not store messages sent to any channel +// with a 34 character id +HK.EPHEMERAL_CHANNEL_LENGTH = 34; + + From 4680de12eecb025ef7e81fc8b484a96456bd8804 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 3 Feb 2020 15:14:52 +0100 Subject: [PATCH 31/53] New UI for the corner popup --- customize.dist/src/less2/include/corner.less | 111 +++++++++++++------ www/common/common-interface.js | 52 ++++++--- www/common/common-ui-elements.js | 98 +++++++++------- www/common/sframe-common.js | 3 + www/pad/inner.js | 6 +- 5 files changed, 177 insertions(+), 93 deletions(-) diff --git a/customize.dist/src/less2/include/corner.less b/customize.dist/src/less2/include/corner.less index 0740586aa..d267f3683 100644 --- a/customize.dist/src/less2/include/corner.less +++ b/customize.dist/src/less2/include/corner.less @@ -4,9 +4,9 @@ --LessLoader_require: LessLoader_currentFile(); }; & { - @corner-button-ok: #2c9b00; - @corner-button-cancel: #990000; @corner-link: #ffff7a; + @corner-blue: @colortheme_logo-1; + @corner-white: @colortheme_base; @keyframes appear { 0% { @@ -27,21 +27,23 @@ .cp-corner-container { position: absolute; - right: 0; - bottom: 0; - width: 300px; - height: 200px; - border-top-left-radius: 200px; - padding: 15px; - text-align: right; - background-color: @colortheme_logo-1; - color: @colortheme_base; + right: 10px; + bottom: 10px; + width: 350px; + padding: 10px; + background-color: @corner-blue; + border: 1px solid @corner-blue; + color: @corner-white; z-index: 9999; transform-origin: bottom right; animation: appear 0.8s ease-in-out; - box-shadow: 0 0 10px 0 @colortheme_logo-1; - //transform: scale(0.1); - //transform: scale(1); + //box-shadow: 0 0 10px 0 @corner-blue; + + &.cp-corner-alt { + background-color: @corner-white; + border: 1px solid @corner-blue; + color: @corner-blue; + } h1, h2, h3 { font-size: 1.5em; @@ -64,7 +66,7 @@ line-height: 15px; display: none; &:hover { - color: darken(@colortheme_base, 15%); + color: darken(@corner-white, 15%); } } .cp-corner-minimize { @@ -86,46 +88,93 @@ } } &.cp-corner-big { - width: 400px; - height: 250px; + width: 500px; + } + + .cp-corner-dontshow { + cursor: pointer; + .fa { + margin-right: 0.3em; + font-size: 1.1em; + } + &:hover { + color: darken(@corner-white, 10%); + } + } + &.cp-corner-alt { + .cp-corner-dontshow { + &:hover { + color: lighten(@corner-blue, 10%); + } + } } .cp-corner-actions { min-height: 30px; - margin: 15px auto; - display: inline-block; + margin: 10px auto; + display: block; + text-align: right; } .cp-corner-footer { - font-style: italic; font-size: 0.8em; } .cp-corner-footer, .cp-corner-text { a { - color: @corner-link; + color: @corner-white; + text-decoration: underline; &:hover { - color: darken(@corner-link, 20%); + color: darken(@corner-white, 10%); } } } + &.cp-corner-alt a { + color: @corner-blue; + &:hover { + color: lighten(@corner-blue, 10%); + } + } button { - border: 0px; padding: 5px; - color: @colortheme_base; - margin-left: 5px; + color: @corner-white; + margin-left: 10px; outline: none; + text-transform: uppercase; + border: 1px solid @corner-white; + .fa, .cptools { + margin-right: 0.3em; + } + &.cp-corner-primary { + background-color: @corner-white; + color: @corner-blue; + &:hover { + background-color: lighten(@corner-blue, 50%); + border-color: lighten(@corner-blue, 50%); + } + } + &.cp-corner-cancel { + background-color: @corner-blue; + color: @corner-white; + &:hover { + background-color: darken(@corner-blue, 10%); + } + } + } + &.cp-corner-alt button { + border-color: @corner-blue; &.cp-corner-primary { - background-color: @corner-button-ok; - font-weight: bold; + background-color: @corner-blue; + color: @corner-white; &:hover { - background-color: lighten(@corner-button-ok, 10%); + background-color: darken(@corner-blue, 10%); + border-color: darken(@corner-blue, 10%); } } &.cp-corner-cancel { - background-color: @corner-button-cancel; - margin-left: 10px; + background-color: @corner-white; + color: @corner-blue; &:hover { - background-color: lighten(@corner-button-cancel, 10%); + background-color: lighten(@corner-blue, 50%); } } } diff --git a/www/common/common-interface.js b/www/common/common-interface.js index 1d681d6e9..04f806d4f 100644 --- a/www/common/common-interface.js +++ b/www/common/common-interface.js @@ -1050,39 +1050,36 @@ define([ return radio; }; + var corner = { + queue: [], + state: false + }; UI.cornerPopup = function (text, actions, footer, opts) { opts = opts || {}; - var minimize = h('div.cp-corner-minimize.fa.fa-window-minimize'); - var maximize = h('div.cp-corner-maximize.fa.fa-window-maximize'); + var dontShowAgain = h('div.cp-corner-dontshow', [ + h('span.fa.fa-times'), + Messages.dontShowAgain || "Don't show again" // XXX + ]); + var popup = h('div.cp-corner-container', [ - minimize, - maximize, - h('div.cp-corner-filler', { style: "width:110px;" }), - h('div.cp-corner-filler', { style: "width:80px;" }), - h('div.cp-corner-filler', { style: "width:60px;" }), - h('div.cp-corner-filler', { style: "width:40px;" }), - h('div.cp-corner-filler', { style: "width:20px;" }), setHTML(h('div.cp-corner-text'), text), h('div.cp-corner-actions', actions), - setHTML(h('div.cp-corner-footer'), footer) + setHTML(h('div.cp-corner-footer'), footer), + opts.dontShowAgain ? dontShowAgain : undefined ]); var $popup = $(popup); - $(minimize).click(function () { - $popup.addClass('cp-minimized'); - }); - $(maximize).click(function () { - $popup.removeClass('cp-minimized'); - }); - if (opts.hidden) { $popup.addClass('cp-minimized'); } if (opts.big) { $popup.addClass('cp-corner-big'); } + if (opts.alt) { + $popup.addClass('cp-corner-alt'); + } var hide = function () { $popup.hide(); @@ -1092,9 +1089,28 @@ define([ }; var deletePopup = function () { $popup.remove(); + if (!corner.queue.length) { + corner.state = false; + return; + } + setTimeout(function () { + $('body').append(corner.queue.pop()); + }, 5000); }; - $('body').append(popup); + $(dontShowAgain).click(function () { + deletePopup(); + if (typeof(opts.dontShowAgain) === "function") { + opts.dontShowAgain(); + } + }); + + if (corner.state) { + corner.queue.push(popup); + } else { + corner.state = true; + $('body').append(popup); + } return { popup: popup, diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index b5e5aa90a..8bfa02e3a 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -4097,52 +4097,68 @@ define([ }; var crowdfundingState = false; - UIElements.displayCrowdfunding = function (common) { + UIElements.displayCrowdfunding = function (common, force) { if (crowdfundingState) { return; } - if (AppConfig.disableCrowdfundingMessages) { return; } var priv = common.getMetadataMgr().getPrivateData(); + + + var todo = function () { + crowdfundingState = true; + // Display the popup + var text = Messages.crowdfunding_popup_text; + var yes = h('button.cp-corner-primary', [ + h('span.fa.fa-external-link'), + 'OpenCollective' + ]); + var no = h('button.cp-corner-cancel', Messages.crowdfunding_popup_no); + var actions = h('div', [no, yes]); + + var dontShowAgain = function () { + common.setAttribute(['general', 'crowdfunding'], false); + Feedback.send('CROWDFUNDING_NEVER'); + }; + + var modal = UI.cornerPopup(text, actions, null, { + big: true, + alt: true, + dontShowAgain: dontShowAgain + }); + + $(yes).click(function () { + modal.delete(); + common.openURL(priv.accounts.donateURL); + Feedback.send('CROWDFUNDING_YES'); + }); + $(modal.popup).find('a').click(function (e) { + e.stopPropagation(); + e.preventDefault(); + modal.delete(); + common.openURL(priv.accounts.donateURL); + Feedback.send('CROWDFUNDING_LINK'); + }); + $(no).click(function () { + modal.delete(); + Feedback.send('CROWDFUNDING_NO'); + }); + }; + + if (force) { + crowdfundingState = true; + return void todo(); + } + + if (AppConfig.disableCrowdfundingMessages) { return; } if (priv.plan) { return; } crowdfundingState = true; - setTimeout(function () { - common.getAttribute(['general', 'crowdfunding'], function (err, val) { - if (err || val === false) { return; } - common.getSframeChannel().query('Q_GET_PINNED_USAGE', null, function (err, obj) { - var quotaMb = obj.quota / (1024 * 1024); - if (quotaMb < 10) { return; } - // Display the popup - var text = Messages.crowdfunding_popup_text; - var yes = h('button.cp-corner-primary', Messages.crowdfunding_popup_yes); - var no = h('button.cp-corner-primary', Messages.crowdfunding_popup_no); - var never = h('button.cp-corner-cancel', Messages.crowdfunding_popup_never); - var actions = h('div', [yes, no, never]); - - var modal = UI.cornerPopup(text, actions, null, {big: true}); - - $(yes).click(function () { - modal.delete(); - common.openURL(priv.accounts.donateURL); - Feedback.send('CROWDFUNDING_YES'); - }); - $(modal.popup).find('a').click(function (e) { - e.stopPropagation(); - e.preventDefault(); - modal.delete(); - common.openURL(priv.accounts.donateURL); - Feedback.send('CROWDFUNDING_LINK'); - }); - $(no).click(function () { - modal.delete(); - Feedback.send('CROWDFUNDING_NO'); - }); - $(never).click(function () { - modal.delete(); - common.setAttribute(['general', 'crowdfunding'], false); - Feedback.send('CROWDFUNDING_NEVER'); - }); - }); + common.getAttribute(['general', 'crowdfunding'], function (err, val) { + if (err || val === false) { return; } + common.getSframeChannel().query('Q_GET_PINNED_USAGE', null, function (err, obj) { + var quotaMb = obj.quota / (1024 * 1024); + if (quotaMb < 10) { return; } + todo(); }); - }, 5000); + }); }; var storePopupState = false; @@ -4164,7 +4180,7 @@ define([ var hide = h('button.cp-corner-cancel', Messages.autostore_hide); var store = h('button.cp-corner-primary', Messages.autostore_store); - var actions = h('div', [store, hide]); + var actions = h('div', [hide, store]); var initialHide = data && data.autoStore && data.autoStore === -1; var modal = UI.cornerPopup(text, actions, footer, {hidden: initialHide}); diff --git a/www/common/sframe-common.js b/www/common/sframe-common.js index 026021d1d..a651bfa31 100644 --- a/www/common/sframe-common.js +++ b/www/common/sframe-common.js @@ -83,6 +83,9 @@ define([ }; // UI + window.CryptPad_UI = UI; + window.CryptPad_UIElements = UIElements; + window.CryptPad_common = funcs; funcs.createUserAdminMenu = callWithCommon(UIElements.createUserAdminMenu); funcs.initFilePicker = callWithCommon(UIElements.initFilePicker); funcs.openFilePicker = callWithCommon(UIElements.openFilePicker); diff --git a/www/pad/inner.js b/www/pad/inner.js index 78f027182..52ae7325a 100644 --- a/www/pad/inner.js +++ b/www/pad/inner.js @@ -736,12 +736,12 @@ define([ }); framework._.sfCommon.isPadStored(function (err, val) { - if (!val) { return; } + //if (!val) { return; } var b64images = $inner.find('img[src^="data:image"]:not(.cke_reset)'); - if (b64images.length && framework._.sfCommon.isLoggedIn()) { + if (true || b64images.length && framework._.sfCommon.isLoggedIn()) { var no = h('button.cp-corner-cancel', Messages.cancel); var yes = h('button.cp-corner-primary', Messages.ok); - var actions = h('div', [yes, no]); + var actions = h('div', [no, yes]); var modal = UI.cornerPopup(Messages.pad_base64, actions, '', {big: true}); $(no).click(function () { modal.delete(); From 06c29ef1d19d46121827c3bc111e6f84db49b750 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 3 Feb 2020 10:03:43 -0500 Subject: [PATCH 32/53] latest api changes to match the netflux-server refactor --- lib/api.js | 17 +++++++++++++---- lib/historyKeeper.js | 35 +++++++++++++++++------------------ 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/lib/api.js b/lib/api.js index d633db59a..c6491d5db 100644 --- a/lib/api.js +++ b/lib/api.js @@ -1,7 +1,7 @@ /* jshint esversion: 6 */ const nThen = require("nthen"); const WebSocketServer = require('ws').Server; -const NetfluxSrv = require('chainpad-server/NetfluxWebsocketSrv'); +const NetfluxSrv = require('chainpad-server'); module.exports.create = function (config) { var rpc; @@ -48,10 +48,19 @@ module.exports.create = function (config) { store: config.store, log: log, }; - // XXX historyKeeper exports a `setConfig` method - var wsSrv = new WebSocketServer(wsConfig); var historyKeeper = HK.create(hkConfig); - NetfluxSrv.run(wsSrv, config, historyKeeper); + + NetfluxSrv.create(new WebSocketServer(wsConfig)) + .on('channelClose', historyKeeper.channelClose) + .on('channelMessage', historyKeeper.channelMessage) + .on('channelOpen', historyKeeper.channelOpen) + .on('sessionClose', function (userId, reason) { + reason = reason; // XXX + }) + .on('error', function (error, label, info) { + info = info; // XXX + }) + .register(historyKeeper.id, historyKeeper.directMessage); }); }; diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index f8d3c8a15..92eb84a5c 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -976,25 +976,24 @@ module.exports.create = function (cfg) { command(ctx, seq, user, parsed); }; - // XXX every one of these values is exported because - // netfluxWebsocketServer needs them to do some magic historyKeeper things - // we could have netflux emit events and let historyKeeper handle them instead return { id: HISTORY_KEEPER_ID, - - // XXX dropChannel allows netflux to clear historyKeeper's cache - // maybe it should emit a 'channel_dropped' event instead - // and let historyKeeper decide what to do - dropChannel: dropChannel, - - // XXX we don't need to export checkExpired if netflux allows it to be HK's responsibility - checkExpired: checkExpired, - - // XXX again, if netflux emitted events then historyKeeper could handle them itself - // and netflux wouldn't need to have historyKeeper-specific code - onDirectMessage: onDirectMessage, - - // XXX same - onChannelMessage: onChannelMessage, + channelMessage: function (ctx, channel, msgStruct) { + onChannelMessage(ctx, channel, msgStruct); + }, + channelClose: function (channelName) { + dropChannel(channelName); + }, + channelOpen: function (ctx, channelName, user) { + ctx.sendMsg(ctx, user, [ + 0, + HISTORY_KEEPER_ID, // ctx.historyKeeper.id + 'JOIN', + channelName + ]); + }, + directMessage: function (ctx, seq, user, json) { + onDirectMessage(ctx, seq, user, json); + }, }; }; From 779e8174432fe0c0a361c1cc5991fc3a4365cf97 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 3 Feb 2020 14:20:05 -0500 Subject: [PATCH 33/53] stop relying on netflux-server internals * create RPC module from inside historyKeeper * stop passing around netflux-server context * update to use newer netflux-server's formal APIs * manage your own cache of indexes instead of storing things in the netflux context --- lib/api.js | 66 ++++++----- lib/commands/admin-rpc.js | 35 ++---- lib/commands/channel.js | 7 +- lib/historyKeeper.js | 236 ++++++++++++++++++++------------------ lib/rpc.js | 38 ++---- 5 files changed, 185 insertions(+), 197 deletions(-) diff --git a/lib/api.js b/lib/api.js index c6491d5db..b66dcb4de 100644 --- a/lib/api.js +++ b/lib/api.js @@ -4,8 +4,6 @@ const WebSocketServer = require('ws').Server; const NetfluxSrv = require('chainpad-server'); module.exports.create = function (config) { - var rpc; - const log = config.log; const wsConfig = { server: config.httpServer, }; @@ -32,35 +30,45 @@ module.exports.create = function (config) { }); }, 1000 * 60 * 5); // run every five minutes })); - }).nThen(function (w) { - require("./rpc").create(config, w(function (e, _rpc) { - if (e) { - w.abort(); - throw e; - } - rpc = _rpc; - })); }).nThen(function () { - var HK = require('./historyKeeper.js'); - var hkConfig = { - tasks: config.tasks, - rpc: rpc, - store: config.store, - log: log, - }; + // asynchronously create a historyKeeper and RPC together + require('./historyKeeper.js').create(config, function (err, historyKeeper) { + if (err) { throw err; } - var historyKeeper = HK.create(hkConfig); + var log = config.log; - NetfluxSrv.create(new WebSocketServer(wsConfig)) - .on('channelClose', historyKeeper.channelClose) - .on('channelMessage', historyKeeper.channelMessage) - .on('channelOpen', historyKeeper.channelOpen) - .on('sessionClose', function (userId, reason) { - reason = reason; // XXX - }) - .on('error', function (error, label, info) { - info = info; // XXX - }) - .register(historyKeeper.id, historyKeeper.directMessage); + // spawn ws server and attach netflux event handlers + NetfluxSrv.create(new WebSocketServer(wsConfig)) + .on('channelClose', historyKeeper.channelClose) + .on('channelMessage', historyKeeper.channelMessage) + .on('channelOpen', historyKeeper.channelOpen) + .on('sessionClose', function (userId, reason) { + if (['BAD_MESSAGE', 'SOCKET_ERROR', 'SEND_MESSAGE_FAIL_2'].indexOf(reason) !== -1) { + return void log.error('SESSION_CLOSE_WITH_ERROR', { + userId: userId, + reason: reason, + }); + } + log.verbose('SESSION_CLOSE_ROUTINE', { + userId: userId, + reason: reason, + }); + }) + .on('error', function (error, label, info) { + if (!error) { return; } + /* labels: + SEND_MESSAGE_FAIL, SEND_MESSAGE_FAIL_2, FAIL_TO_DISCONNECT, + FAIL_TO_TERMINATE, HANDLE_CHANNEL_LEAVE, NETFLUX_BAD_MESSAGE, + NETFLUX_WEBSOCKET_ERROR + */ + log.error(label, { + code: error.code, + message: error.message, + stack: error.stack, + info: info, + }); + }) + .register(historyKeeper.id, historyKeeper.directMessage); + }); }); }; diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 24a8e4348..763bfb517 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -6,25 +6,15 @@ var Fs = require("fs"); var Admin = module.exports; -var getActiveSessions = function (Env, ctx, cb) { - var total = ctx.users ? Object.keys(ctx.users).length : '?'; - - var ips = []; - Object.keys(ctx.users).forEach(function (u) { - var user = ctx.users[u]; - var socket = user.socket; - var req = socket.upgradeReq; - var conn = req && req.connection; - var ip = (req && req.headers && req.headers['x-forwarded-for']) || (conn && conn.remoteAddress); - if (ip && ips.indexOf(ip) === -1) { - ips.push(ip); - } - }); - - cb (void 0, [total, ips.length]); +var getActiveSessions = function (Env, Server, cb) { + var stats = Server.getSessionStats(); + cb(void 0, [ + stats.total, + stats.unique + ]); }; -var shutdown = function (Env, ctx, cb) { +var shutdown = function (Env, Server, cb) { return void cb('E_NOT_IMPLEMENTED'); //clearInterval(Env.sessionExpirationInterval); // XXX set a flag to prevent incoming database writes @@ -91,19 +81,18 @@ var getDiskUsage = function (Env, cb) { }); }; - - -Admin.command = function (Env, ctx, publicKey, data, cb) { +Admin.command = function (Env, Server, publicKey, data, cb) { var admins = Env.admins; if (admins.indexOf(publicKey) === -1) { return void cb("FORBIDDEN"); } + // Handle commands here switch (data[0]) { case 'ACTIVE_SESSIONS': - return getActiveSessions(Env, ctx, cb); + return getActiveSessions(Env, Server, cb); case 'ACTIVE_PADS': - return cb(void 0, ctx.channels ? Object.keys(ctx.channels).length : '?'); + return cb(void 0, Server.getActiveChannelCount()); case 'REGISTERED_USERS': return getRegisteredUsers(Env, cb); case 'DISK_USAGE': @@ -112,7 +101,7 @@ Admin.command = function (Env, ctx, publicKey, data, cb) { Env.flushCache(); return cb(void 0, true); case 'SHUTDOWN': - return shutdown(Env, ctx, cb); + return shutdown(Env, Server, cb); default: return cb('UNHANDLED_ADMIN_COMMAND'); } diff --git a/lib/commands/channel.js b/lib/commands/channel.js index 052aa3c44..f232ccea8 100644 --- a/lib/commands/channel.js +++ b/lib/commands/channel.js @@ -147,7 +147,7 @@ Channel.isNewChannel = function (Env, channel, cb) { Otherwise behaves the same as sending to a channel */ -Channel.writePrivateMessage = function (Env, args, nfwssCtx, cb) { +Channel.writePrivateMessage = function (Env, args, Server, cb) { var channelId = args[0]; var msg = args[1]; @@ -161,7 +161,7 @@ Channel.writePrivateMessage = function (Env, args, nfwssCtx, cb) { // We expect a modern netflux-websocket-server instance // if this API isn't here everything will fall apart anyway - if (!(nfwssCtx && nfwssCtx.historyKeeper && typeof(nfwssCtx.historyKeeper.onChannelMessage) === 'function')) { + if (!(Server && typeof(Server.send) === 'function')) { return void cb("NOT_IMPLEMENTED"); } @@ -180,8 +180,9 @@ Channel.writePrivateMessage = function (Env, args, nfwssCtx, cb) { msg // the actual message content. Generally a string ]; + // XXX this API doesn't exist anymore... // store the message and do everything else that is typically done when going through historyKeeper - nfwssCtx.historyKeeper.onChannelMessage(nfwssCtx, channelStruct, fullMessage); + Env.historyKeeper.onChannelMessage(Server, channelStruct, fullMessage); // call back with the message and the target channel. // historyKeeper will take care of broadcasting it if anyone is in the channel diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 92eb84a5c..30500f180 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -9,6 +9,8 @@ const Meta = require("./metadata"); const WriteQueue = require("./write-queue"); const BatchRead = require("./batch-read"); +const RPC = require("./rpc"); + const Extras = require("./hk-util.js"); const STANDARD_CHANNEL_LENGTH = Extras.STANDARD_CHANNEL_LENGTH; const EPHEMERAL_CHANNEL_LENGTH = Extras.EPHEMERAL_CHANNEL_LENGTH; @@ -66,8 +68,8 @@ const isValidValidateKeyString = function (key) { var CHECKPOINT_PATTERN = /^cp\|(([A-Za-z0-9+\/=]+)\|)?/; -module.exports.create = function (cfg) { - const rpc = cfg.rpc; +module.exports.create = function (cfg, cb) { + var rpc; const tasks = cfg.tasks; const store = cfg.store; Log = cfg.log; @@ -75,6 +77,7 @@ module.exports.create = function (cfg) { Log.silly('HK_LOADING', 'LOADING HISTORY_KEEPER MODULE'); const metadata_cache = {}; + const channel_cache = {}; const HISTORY_KEEPER_ID = Crypto.randomBytes(8).toString('hex'); Log.verbose('HK_ID', 'History keeper ID: ' + HISTORY_KEEPER_ID); @@ -211,8 +214,9 @@ module.exports.create = function (cfg) { if the channel exists but its index does not then it caches the index */ const batchIndexReads = BatchRead("HK_GET_INDEX"); - const getIndex = (ctx, channelName, cb) => { - const chan = ctx.channels[channelName]; + const getIndex = (channelName, cb) => { + const chan = channel_cache[channelName]; + // if there is a channel in memory and it has an index cached, return it if (chan && chan.index) { // enforce async behaviour @@ -233,15 +237,7 @@ module.exports.create = function (cfg) { }); }; - /*:: - type cp_index_item = { - offset: number, - line: number - } - */ - /* storeMessage - * ctx * channel id * the message to store * whether the message is a checkpoint @@ -260,7 +256,7 @@ module.exports.create = function (cfg) { */ const queueStorage = WriteQueue(); - const storeMessage = function (ctx, channel, msg, isCp, optionalMessageHash) { + const storeMessage = function (channel, msg, isCp, optionalMessageHash) { const id = channel.id; queueStorage(id, function (next) { @@ -284,7 +280,7 @@ module.exports.create = function (cfg) { } })); }).nThen((waitFor) => { - getIndex(ctx, id, waitFor((err, index) => { + getIndex(id, waitFor((err, index) => { if (err) { Log.warn("HK_STORE_MESSAGE_INDEX", err.stack); // non-critical, we'll be able to get the channel index later @@ -298,10 +294,10 @@ module.exports.create = function (cfg) { delete index.offsetByHash[k]; } } - index.cpIndex.push(({ + index.cpIndex.push({ offset: index.size, line: ((index.line || 0) + 1) - } /*:cp_index_item*/)); + }); } if (optionalMessageHash) { index.offsetByHash[optionalMessageHash] = index.size; } index.size += msgBin.length; @@ -313,21 +309,10 @@ module.exports.create = function (cfg) { }); }; - /* historyKeeperBroadcast - * uses API from the netflux server to send messages to every member of a channel - * sendMsg runs in a try-catch and drops users if sending a message fails - */ - const historyKeeperBroadcast = function (ctx, channel, msg) { - let chan = ctx.channels[channel] || (([] /*:any*/) /*:Chan_t*/); - chan.forEach(function (user) { - ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)]); - }); - }; - /* expireChannel is here to clean up channels that should have been removed but for some reason are still present */ - const expireChannel = function (ctx, channel) { + const expireChannel = function (channel) { return void store.archiveChannel(channel, function (err) { Log.info("ARCHIVAL_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", { channelId: channel, @@ -336,6 +321,14 @@ module.exports.create = function (cfg) { }); }; + /* dropChannel + * cleans up memory structures which are managed entirely by the historyKeeper + */ + const dropChannel = function (chanName) { + delete metadata_cache[chanName]; + delete channel_cache[chanName]; + }; + /* checkExpired * synchronously returns true or undefined to indicate whether the channel is expired * according to its metadata @@ -347,7 +340,7 @@ module.exports.create = function (cfg) { FIXME the boolean nature of this API should be separated from its side effects */ - const checkExpired = function (ctx, channel) { + const checkExpired = function (Server, channel) { if (!(channel && channel.length === STANDARD_CHANNEL_LENGTH)) { return false; } let metadata = metadata_cache[channel]; if (!(metadata && typeof(metadata.expire) === 'number')) { return false; } @@ -362,18 +355,16 @@ module.exports.create = function (cfg) { // there may have been a problem with scheduling tasks // or the scheduled tasks may not be running // so trigger a removal from here - if (pastDue >= ONE_DAY) { expireChannel(ctx, channel); } + if (pastDue >= ONE_DAY) { expireChannel(channel); } // close the channel store.closeChannel(channel, function () { - historyKeeperBroadcast(ctx, channel, { + // XXX make sure that clients actually disconnect when we broadcast an error + Server.channelBroadcast(channel, { error: 'EEXPIRED', channel: channel - }); - // remove it from any caches after you've told anyone in the channel - // that it has expired - delete ctx.channels[channel]; - delete metadata_cache[channel]; + }, HISTORY_KEEPER_ID); + dropChannel(channel); }); // return true to indicate that it has expired @@ -391,7 +382,7 @@ module.exports.create = function (cfg) { * adds timestamps to incoming messages * writes messages to the store */ - const onChannelMessage = function (ctx, channel, msgStruct) { + const onChannelMessage = function (Server, channel, msgStruct) { // TODO our usage of 'channel' here looks prone to errors // we only use it for its 'id', but it can contain other stuff // also, we're using this RPC from both the RPC and Netflux-server @@ -414,7 +405,7 @@ module.exports.create = function (cfg) { let metadata; nThen(function (w) { // getIndex (and therefore the latest metadata) - getIndex(ctx, channel.id, w(function (err, index) { + getIndex(channel.id, w(function (err, index) { if (err) { w.abort(); return void Log.error('CHANNEL_MESSAGE_ERROR', err); @@ -429,7 +420,7 @@ module.exports.create = function (cfg) { metadata = index.metadata; // don't write messages to expired channels - if (checkExpired(ctx, channel)) { return void w.abort(); } + if (checkExpired(Server, channel)) { return void w.abort(); } // if there's no validateKey present skip to the next block if (!metadata.validateKey) { return; } @@ -479,20 +470,10 @@ module.exports.create = function (cfg) { msgStruct.push(now()); // storeMessage - storeMessage(ctx, channel, JSON.stringify(msgStruct), isCp, getHash(msgStruct[4], Log)); + storeMessage(channel, JSON.stringify(msgStruct), isCp, getHash(msgStruct[4], Log)); }); }; - /* dropChannel - * exported as API - * used by chainpad-server/NetfluxWebsocketSrv.js - * cleans up memory structures which are managed entirely by the historyKeeper - * the netflux server manages other memory in ctx.channels - */ - const dropChannel = function (chanName) { - delete metadata_cache[chanName]; - }; - /* getHistoryOffset returns a number representing the byte offset from the start of the log for whatever history you're seeking. @@ -522,12 +503,12 @@ module.exports.create = function (cfg) { * -1 if you didn't find it */ - const getHistoryOffset = (ctx, channelName, lastKnownHash, cb /*:(e:?Error, os:?number)=>void*/) => { + const getHistoryOffset = (channelName, lastKnownHash, cb) => { // lastKnownhash === -1 means we want the complete history if (lastKnownHash === -1) { return void cb(null, 0); } let offset = -1; nThen((waitFor) => { - getIndex(ctx, channelName, waitFor((err, index) => { + getIndex(channelName, waitFor((err, index) => { if (err) { waitFor.abort(); return void cb(err); } // check if the "hash" the client is requesting exists in the index @@ -600,10 +581,10 @@ module.exports.create = function (cfg) { * GET_HISTORY */ - const getHistoryAsync = (ctx, channelName, lastKnownHash, beforeHash, handler, cb) => { + const getHistoryAsync = (channelName, lastKnownHash, beforeHash, handler, cb) => { let offset = -1; nThen((waitFor) => { - getHistoryOffset(ctx, channelName, lastKnownHash, waitFor((err, os) => { + getHistoryOffset(channelName, lastKnownHash, waitFor((err, os) => { if (err) { waitFor.abort(); return void cb(err); @@ -666,42 +647,50 @@ module.exports.create = function (cfg) { /* onChannelCleared * broadcasts to all clients in a channel if that channel is deleted */ - const onChannelCleared = function (ctx, channel) { - historyKeeperBroadcast(ctx, channel, { + const onChannelCleared = function (Server, channel) { + Server.channelBroadcast(channel, { error: 'ECLEARED', channel: channel - }); + }, HISTORY_KEEPER_ID); }; + // When a channel is removed from datastore, broadcast a message to all its connected users - const onChannelDeleted = function (ctx, channel) { + const onChannelDeleted = function (Server, channel) { store.closeChannel(channel, function () { - historyKeeperBroadcast(ctx, channel, { + Server.channelBroadcast(channel, { error: 'EDELETED', channel: channel - }); + }, HISTORY_KEEPER_ID); }); - delete ctx.channels[channel]; + + delete channel_cache[channel]; + Server.clearChannel(channel); delete metadata_cache[channel]; }; // Check if the selected channel is expired // If it is, remove it from memory and broadcast a message to its members - const onChannelMetadataChanged = function (ctx, channel, metadata) { - if (channel && metadata_cache[channel] && typeof (metadata) === "object") { - Log.silly('SET_METADATA_CACHE', 'Channel '+ channel +', metadata: '+ JSON.stringify(metadata)); - metadata_cache[channel] = metadata; - if (ctx.channels[channel] && ctx.channels[channel].index) { - ctx.channels[channel].index.metadata = metadata; - } - historyKeeperBroadcast(ctx, channel, metadata); + const onChannelMetadataChanged = function (Server, channel, metadata) { + if (!(channel && metadata_cache[channel] && typeof (metadata) === "object")) { return; } + Log.silly('SET_METADATA_CACHE', { + channel: channel, + metadata: JSON.stringify(metadata), + }); + + metadata_cache[channel] = metadata; + + if (channel_cache[channel] && channel_cache[channel].index) { + channel_cache[channel].index.metadata = metadata; } + Server.channelBroadcast(channel, metadata, HISTORY_KEEPER_ID); }; - const handleGetHistory = function (ctx, seq, user, parsed) { + const handleGetHistory = function (Server, seq, user, parsed) { // parsed[1] is the channel id // parsed[2] is a validation key or an object containing metadata (optionnal) // parsed[3] is the last known hash (optionnal) - ctx.sendMsg(ctx, user, [seq, 'ACK']); + + Server.send(user.id, [seq, 'ACK']); var channelName = parsed[1]; var config = parsed[2]; var metadata = {}; @@ -736,7 +725,7 @@ module.exports.create = function (cfg) { unfortunately, we can't just serve it blindly, since then young channels will send the metadata twice, so let's do a quick check of what we're going to serve... */ - getIndex(ctx, channelName, waitFor((err, index) => { + getIndex(channelName, waitFor((err, index) => { /* if there's an error here, it should be encountered and handled by the next nThen block. so, let's just fall through... @@ -750,29 +739,29 @@ module.exports.create = function (cfg) { if (!index || !index.metadata) { return void w(); } // And then check if the channel is expired. If it is, send the error and abort // FIXME this is hard to read because 'checkExpired' has side effects - if (checkExpired(ctx, channelName)) { return void waitFor.abort(); } + if (checkExpired(Server, channelName)) { return void waitFor.abort(); } // always send metadata with GET_HISTORY requests - ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); + Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); })); }).nThen(() => { let msgCount = 0; // TODO compute lastKnownHash in a manner such that it will always skip past the metadata line? - getHistoryAsync(ctx, channelName, lastKnownHash, false, (msg, readMore) => { + getHistoryAsync(channelName, lastKnownHash, false, (msg, readMore) => { if (!msg) { return; } msgCount++; // avoid sending the metadata message a second time if (isMetadataMessage(msg) && metadata_cache[channelName]) { return readMore(); } - ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)], readMore); + Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)], readMore); }, (err) => { if (err && err.code !== 'ENOENT') { if (err.message !== 'EINVAL') { Log.error("HK_GET_HISTORY", err); } const parsedMsg = {error:err.message, channel: channelName}; - ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); return; } - const chan = ctx.channels[channelName]; + const chan = channel_cache[channelName]; if (msgCount === 0 && !metadata_cache[channelName] && chan && chan.indexOf(user) > -1) { metadata_cache[channelName] = metadata; @@ -811,21 +800,22 @@ module.exports.create = function (cfg) { } }); } - ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(metadata)]); + Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(metadata)]); } // End of history message: let parsedMsg = {state: 1, channel: channelName}; - ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + + Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); }); }); }; - const handleGetHistoryRange = function (ctx, seq, user, parsed) { + const handleGetHistoryRange = function (Server, seq, user, parsed) { var channelName = parsed[1]; var map = parsed[2]; if (!(map && typeof(map) === 'object')) { - return void ctx.sendMsg(ctx, user, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]); + return void Server.send(user.id, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]); } var oldestKnownHash = map.from; @@ -833,14 +823,14 @@ module.exports.create = function (cfg) { var desiredCheckpoint = map.cpCount; var txid = map.txid; if (typeof(desiredMessages) !== 'number' && typeof(desiredCheckpoint) !== 'number') { - return void ctx.sendMsg(ctx, user, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]); + return void Server.send(user.id, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]); } if (!txid) { - return void ctx.sendMsg(ctx, user, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]); + return void Server.send(user.id, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]); } - ctx.sendMsg(ctx, user, [seq, 'ACK']); + Server.send(user.id, [seq, 'ACK']); return void getOlderHistory(channelName, oldestKnownHash, function (messages) { var toSend = []; if (typeof (desiredMessages) === "number") { @@ -856,64 +846,66 @@ module.exports.create = function (cfg) { } } toSend.forEach(function (msg) { - ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, + Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['HISTORY_RANGE', txid, msg])]); }); - ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, + Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['HISTORY_RANGE_END', txid, channelName]) ]); }); }; - const handleGetFullHistory = function (ctx, seq, user, parsed) { + const handleGetFullHistory = function (Server, seq, user, parsed) { // parsed[1] is the channel id // parsed[2] is a validation key (optionnal) // parsed[3] is the last known hash (optionnal) - ctx.sendMsg(ctx, user, [seq, 'ACK']); + + Server.send(user.id, [seq, 'ACK']); // FIXME should we send metadata here too? // none of the clientside code which uses this API needs metadata, but it won't hurt to send it (2019-08-22) - return void getHistoryAsync(ctx, parsed[1], -1, false, (msg, readMore) => { + return void getHistoryAsync(parsed[1], -1, false, (msg, readMore) => { if (!msg) { return; } - ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['FULL_HISTORY', msg])], readMore); + Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['FULL_HISTORY', msg])], readMore); }, (err) => { let parsedMsg = ['FULL_HISTORY_END', parsed[1]]; if (err) { Log.error('HK_GET_FULL_HISTORY', err.stack); parsedMsg = ['ERROR', parsed[1], err.message]; } - ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); }); }; - const handleRPC = function (ctx, seq, user, parsed) { + const handleRPC = function (Server, seq, user, parsed) { if (typeof(rpc) !== 'function') { return; } /* RPC Calls... */ var rpc_call = parsed.slice(1); - ctx.sendMsg(ctx, user, [seq, 'ACK']); + // XXX ensure user is guaranteed to have 'id' + Server.send(user.id, [seq, 'ACK']); try { // slice off the sequence number and pass in the rest of the message - rpc(ctx, rpc_call, function (err, output) { + rpc(Server, rpc_call, function (err, output) { if (err) { - ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', err])]); + Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', err])]); return; } var msg = rpc_call[0].slice(); if (msg[3] === 'REMOVE_OWNED_CHANNEL') { - onChannelDeleted(ctx, msg[4]); + onChannelDeleted(Server, msg[4]); } if (msg[3] === 'CLEAR_OWNED_CHANNEL') { - onChannelCleared(ctx, msg[4]); + onChannelCleared(Server, msg[4]); } if (msg[3] === 'SET_METADATA') { // or whatever we call the RPC???? // make sure we update our cache of metadata // or at least invalidate it and force other mechanisms to recompute its state // 'output' could be the new state as computed by rpc - onChannelMetadataChanged(ctx, msg[4].channel, output[1]); + onChannelMetadataChanged(Server, msg[4].channel, output[1]); } // unauthenticated RPC calls have a different message format @@ -921,10 +913,10 @@ module.exports.create = function (cfg) { // this is an inline reimplementation of historyKeeperBroadcast // because if we use that directly it will bypass signature validation // which opens up the user to malicious behaviour - let chan = ctx.channels[output.channel]; + let chan = channel_cache[output.channel]; if (chan && chan.length) { chan.forEach(function (user) { - ctx.sendMsg(ctx, user, output.message); + Server.send(user.id, output.message); //[0, null, 'MSG', user.id, JSON.stringify(output.message)]); }); } @@ -934,10 +926,10 @@ module.exports.create = function (cfg) { } // finally, send a response to the client that sent the RPC - ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0]].concat(output))]); + Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0]].concat(output))]); }); } catch (e) { - ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]); + Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]); } }; @@ -953,7 +945,7 @@ module.exports.create = function (cfg) { * check if it's expired and execute all the associated side-effects * routes queries to the appropriate handlers */ - const onDirectMessage = function (ctx, seq, user, json) { + const onDirectMessage = function (Server, seq, user, json) { Log.silly('HK_MESSAGE', json); let parsed; @@ -967,33 +959,49 @@ module.exports.create = function (cfg) { // If the requested history is for an expired channel, abort // Note the if we don't have the keys for that channel in metadata_cache, we'll // have to abort later (once we know the expiration time) - if (checkExpired(ctx, parsed[1])) { return; } + if (checkExpired(Server, parsed[1])) { return; } // look up the appropriate command in the map of commands or fall back to RPC var command = directMessageCommands[parsed[0]] || handleRPC; // run the command with the standard function signature - command(ctx, seq, user, parsed); + command(Server, seq, user, parsed); }; - return { + cfg.historyKeeper = { id: HISTORY_KEEPER_ID, - channelMessage: function (ctx, channel, msgStruct) { - onChannelMessage(ctx, channel, msgStruct); + + channelMessage: function (Server, channel, msgStruct) { + // netflux-server emits 'channelMessage' events whenever someone broadcasts to a channel + // historyKeeper stores these messages if the channel id indicates that they are + // a channel type with permanent history + onChannelMessage(Server, channel, msgStruct); }, channelClose: function (channelName) { + // netflux-server emits 'channelClose' events whenever everyone leaves a channel + // we drop cached metadata and indexes at the same time dropChannel(channelName); }, - channelOpen: function (ctx, channelName, user) { - ctx.sendMsg(ctx, user, [ + channelOpen: function (Server, channelName, userId) { + channel_cache[channelName] = {}; + Server.send(userId, [ 0, - HISTORY_KEEPER_ID, // ctx.historyKeeper.id + HISTORY_KEEPER_ID, 'JOIN', channelName ]); }, - directMessage: function (ctx, seq, user, json) { - onDirectMessage(ctx, seq, user, json); + directMessage: function (Server, seq, user, json) { + // netflux-server allows you to register an id with a handler + // this handler is invoked every time someone sends a message to that id + onDirectMessage(Server, seq, user, json); }, }; + + RPC.create(cfg, function (err, _rpc) { + if (err) { throw err; } + + rpc = _rpc; + cb(void 0, cfg.historyKeeper); + }); }; diff --git a/lib/rpc.js b/lib/rpc.js index 171fb590f..47ea9976f 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -24,7 +24,6 @@ const UNAUTHENTICATED_CALLS = [ 'GET_MULTIPLE_FILE_SIZE', 'IS_CHANNEL_PINNED', 'IS_NEW_CHANNEL', - 'GET_HISTORY_OFFSET', 'GET_DELETED_PADS', 'WRITE_PRIVATE_MESSAGE', ]; @@ -66,25 +65,9 @@ var isUnauthenticateMessage = function (msg) { return msg && msg.length === 2 && isUnauthenticatedCall(msg[0]); }; -var handleUnauthenticatedMessage = function (Env, msg, respond, nfwssCtx) { +var handleUnauthenticatedMessage = function (Env, msg, respond, Server) { Env.Log.silly('LOG_RPC', msg[0]); switch (msg[0]) { - case 'GET_HISTORY_OFFSET': { // XXX not actually used anywhere? - if (typeof(msg[1]) !== 'object' || typeof(msg[1].channelName) !== 'string') { - return respond('INVALID_ARG_FORMAT', msg); - } - const msgHash = typeof(msg[1].msgHash) === 'string' ? msg[1].msgHash : undefined; - nfwssCtx.getHistoryOffset(nfwssCtx, msg[1].channelName, msgHash, (e, ret) => { - if (e) { - if (e.code !== 'ENOENT') { - Env.WARN(e.stack, msg); - } - return respond(e.message); - } - respond(e, [null, ret, null]); - }); - break; - } case 'GET_FILE_SIZE': return void Pinning.getFileSize(Env, msg[1], function (e, size) { Env.WARN(e, msg[1]); @@ -120,7 +103,7 @@ var handleUnauthenticatedMessage = function (Env, msg, respond, nfwssCtx) { respond(e, [null, isNew, null]); }); case 'WRITE_PRIVATE_MESSAGE': - return void Channel.writePrivateMessage(Env, msg[1], nfwssCtx, function (e, output) { + return void Channel.writePrivateMessage(Env, msg[1], Server, function (e, output) { respond(e, output); }); default: @@ -134,7 +117,7 @@ var handleAuthenticatedMessage = function (Env, map) { var safeKey = map.safeKey; var publicKey = map.publicKey; var Respond = map.Respond; - var ctx = map.ctx; + var Server = map.Server; Env.Log.silly('LOG_RPC', msg[0]); switch (msg[0]) { @@ -265,7 +248,7 @@ var handleAuthenticatedMessage = function (Env, map) { Respond(e); }); case 'ADMIN': - return void Admin.command(Env, ctx, safeKey, msg[1], function (e, result) { // XXX SPECIAL + return void Admin.command(Env, Server, safeKey, msg[1], function (e, result) { // XXX SPECIAL if (e) { Env.WARN(e, result); return void Respond(e); @@ -285,7 +268,7 @@ var handleAuthenticatedMessage = function (Env, map) { } }; -var rpc = function (Env, ctx, data, respond) { +var rpc = function (Env, Server, data, respond) { if (!Array.isArray(data)) { Env.Log.debug('INVALID_ARG_FORMET', data); return void respond('INVALID_ARG_FORMAT'); @@ -304,7 +287,7 @@ var rpc = function (Env, ctx, data, respond) { } if (isUnauthenticateMessage(msg)) { - return handleUnauthenticatedMessage(Env, msg, respond, ctx); + return handleUnauthenticatedMessage(Env, msg, respond); } var signature = msg.shift(); @@ -369,7 +352,7 @@ var rpc = function (Env, ctx, data, respond) { safeKey: safeKey, publicKey: publicKey, Respond: Respond, - ctx: ctx, + Server: Server, }); }; @@ -394,6 +377,7 @@ RPC.create = function (config, cb) { }; var Env = { + historyKeeper: config.historyKeeper, defaultStorageLimit: config.defaultStorageLimit, maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024), Sessions: {}, @@ -465,11 +449,9 @@ RPC.create = function (config, cb) { Env.blobStore = blob; })); }).nThen(function () { - // XXX it's ugly that we pass ctx and Env separately - // when they're effectively the same thing... - cb(void 0, function (ctx, data, respond) { + cb(void 0, function (Server, data, respond) { try { - return rpc(Env, ctx, data, respond); + return rpc(Env, Server, data, respond); } catch (e) { console.log("Error from RPC with data " + JSON.stringify(data)); console.log(e.stack); From e3269df7f0dea1b0f60da469547f411afc6be5f0 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 3 Feb 2020 14:23:31 -0500 Subject: [PATCH 34/53] fix some critical errors in the trim history storage api --- storage/file.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/storage/file.js b/storage/file.js index 08ec98f85..09b743ace 100644 --- a/storage/file.js +++ b/storage/file.js @@ -955,7 +955,9 @@ var trimChannel = function (env, channelName, hash, _cb) { var handler = function (msgObj, readMore, abort) { if (ABORT) { return void abort(); } // the first message might be metadata... ignore it if so - if (i++ === 0 && msgObj.buff.indexOf('{') === 0) { return; } + if (i++ === 0 && msgObj.buff.indexOf('{') === 0) { + return readMore(); + } if (retain) { // if this flag is set then you've already found @@ -973,6 +975,9 @@ var trimChannel = function (env, channelName, hash, _cb) { if (msgHash === hash) { // everything from this point on should be retained retain = true; + return void tempStream.write(msgObj.buff, function () { + readMore(); + }); } }; From 6523974ca2389c005e79e2a46aa2da05c4451ac2 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 3 Feb 2020 15:47:41 -0500 Subject: [PATCH 35/53] fix a WRITE_PRIVATE_MESSAGE rpc regression --- lib/commands/channel.js | 6 ++-- lib/historyKeeper.js | 77 ++++++++++++++++++++--------------------- lib/rpc.js | 2 +- 3 files changed, 42 insertions(+), 43 deletions(-) diff --git a/lib/commands/channel.js b/lib/commands/channel.js index f232ccea8..ec4bf3fe6 100644 --- a/lib/commands/channel.js +++ b/lib/commands/channel.js @@ -180,9 +180,9 @@ Channel.writePrivateMessage = function (Env, args, Server, cb) { msg // the actual message content. Generally a string ]; - // XXX this API doesn't exist anymore... - // store the message and do everything else that is typically done when going through historyKeeper - Env.historyKeeper.onChannelMessage(Server, channelStruct, fullMessage); + // historyKeeper already knows how to handle metadata and message validation, so we just pass it off here + // if the message isn't valid it won't be stored. + Env.historyKeeper.channelMessage(Server, channelStruct, fullMessage); // call back with the message and the target channel. // historyKeeper will take care of broadcasting it if anyone is in the channel diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 30500f180..d5202876d 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -685,12 +685,12 @@ module.exports.create = function (cfg, cb) { Server.channelBroadcast(channel, metadata, HISTORY_KEEPER_ID); }; - const handleGetHistory = function (Server, seq, user, parsed) { + const handleGetHistory = function (Server, seq, userId, parsed) { // parsed[1] is the channel id // parsed[2] is a validation key or an object containing metadata (optionnal) // parsed[3] is the last known hash (optionnal) - Server.send(user.id, [seq, 'ACK']); + Server.send(userId, [seq, 'ACK']); var channelName = parsed[1]; var config = parsed[2]; var metadata = {}; @@ -741,7 +741,7 @@ module.exports.create = function (cfg, cb) { // FIXME this is hard to read because 'checkExpired' has side effects if (checkExpired(Server, channelName)) { return void waitFor.abort(); } // always send metadata with GET_HISTORY requests - Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); + Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(index.metadata)], w); })); }).nThen(() => { let msgCount = 0; @@ -752,18 +752,18 @@ module.exports.create = function (cfg, cb) { msgCount++; // avoid sending the metadata message a second time if (isMetadataMessage(msg) && metadata_cache[channelName]) { return readMore(); } - Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)], readMore); + Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(msg)], readMore); }, (err) => { if (err && err.code !== 'ENOENT') { if (err.message !== 'EINVAL') { Log.error("HK_GET_HISTORY", err); } const parsedMsg = {error:err.message, channel: channelName}; - Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(parsedMsg)]); return; } const chan = channel_cache[channelName]; - if (msgCount === 0 && !metadata_cache[channelName] && chan && chan.indexOf(user) > -1) { + if (msgCount === 0 && !metadata_cache[channelName] && Server.channelContainsUser(channelName, userId)) { metadata_cache[channelName] = metadata; // the index will have already been constructed and cached at this point @@ -800,22 +800,23 @@ module.exports.create = function (cfg, cb) { } }); } - Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(metadata)]); + Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(metadata)]); } // End of history message: let parsedMsg = {state: 1, channel: channelName}; - Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(parsedMsg)]); }); }); }; - const handleGetHistoryRange = function (Server, seq, user, parsed) { + const handleGetHistoryRange = function (Server, seq, userId, parsed) { var channelName = parsed[1]; var map = parsed[2]; + if (!(map && typeof(map) === 'object')) { - return void Server.send(user.id, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]); + return void Server.send(userId, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]); } var oldestKnownHash = map.from; @@ -823,14 +824,14 @@ module.exports.create = function (cfg, cb) { var desiredCheckpoint = map.cpCount; var txid = map.txid; if (typeof(desiredMessages) !== 'number' && typeof(desiredCheckpoint) !== 'number') { - return void Server.send(user.id, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]); + return void Server.send(userId, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]); } if (!txid) { - return void Server.send(user.id, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]); + return void Server.send(userId, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]); } - Server.send(user.id, [seq, 'ACK']); + Server.send(userId, [seq, 'ACK']); return void getOlderHistory(channelName, oldestKnownHash, function (messages) { var toSend = []; if (typeof (desiredMessages) === "number") { @@ -846,51 +847,50 @@ module.exports.create = function (cfg, cb) { } } toSend.forEach(function (msg) { - Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, + Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(['HISTORY_RANGE', txid, msg])]); }); - Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, + Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(['HISTORY_RANGE_END', txid, channelName]) ]); }); }; - const handleGetFullHistory = function (Server, seq, user, parsed) { + const handleGetFullHistory = function (Server, seq, userId, parsed) { // parsed[1] is the channel id // parsed[2] is a validation key (optionnal) // parsed[3] is the last known hash (optionnal) - Server.send(user.id, [seq, 'ACK']); + Server.send(userId, [seq, 'ACK']); // FIXME should we send metadata here too? // none of the clientside code which uses this API needs metadata, but it won't hurt to send it (2019-08-22) return void getHistoryAsync(parsed[1], -1, false, (msg, readMore) => { if (!msg) { return; } - Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['FULL_HISTORY', msg])], readMore); + Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(['FULL_HISTORY', msg])], readMore); }, (err) => { let parsedMsg = ['FULL_HISTORY_END', parsed[1]]; if (err) { Log.error('HK_GET_FULL_HISTORY', err.stack); parsedMsg = ['ERROR', parsed[1], err.message]; } - Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(parsedMsg)]); }); }; - const handleRPC = function (Server, seq, user, parsed) { + const handleRPC = function (Server, seq, userId, parsed) { if (typeof(rpc) !== 'function') { return; } /* RPC Calls... */ var rpc_call = parsed.slice(1); - // XXX ensure user is guaranteed to have 'id' - Server.send(user.id, [seq, 'ACK']); + Server.send(userId, [seq, 'ACK']); try { // slice off the sequence number and pass in the rest of the message rpc(Server, rpc_call, function (err, output) { if (err) { - Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', err])]); + Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify([parsed[0], 'ERROR', err])]); return; } var msg = rpc_call[0].slice(); @@ -910,26 +910,25 @@ module.exports.create = function (cfg, cb) { // unauthenticated RPC calls have a different message format if (msg[0] === "WRITE_PRIVATE_MESSAGE" && output && output.channel) { - // this is an inline reimplementation of historyKeeperBroadcast - // because if we use that directly it will bypass signature validation - // which opens up the user to malicious behaviour - let chan = channel_cache[output.channel]; - if (chan && chan.length) { - chan.forEach(function (user) { - Server.send(user.id, output.message); - //[0, null, 'MSG', user.id, JSON.stringify(output.message)]); - }); - } + // clients don't validate messages sent by the historyKeeper + // so this broadcast needs to come from a different id + // we pass 'null' to indicate that it's not coming from a real user + // to ensure that they know not to trust this message + Server.getChannelUserList(output.channel).forEach(function (userId) { + Server.send(userId, output.message); + }); + // rpc and anonRpc expect their responses to be of a certain length // and we've already used the output of the rpc call, so overwrite it output = [null, null, null]; } // finally, send a response to the client that sent the RPC - Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0]].concat(output))]); + Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify([parsed[0]].concat(output))]); }); } catch (e) { - Server.send(user.id, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]); + // if anything throws in the middle, send an error + Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]); } }; @@ -945,7 +944,7 @@ module.exports.create = function (cfg, cb) { * check if it's expired and execute all the associated side-effects * routes queries to the appropriate handlers */ - const onDirectMessage = function (Server, seq, user, json) { + const onDirectMessage = function (Server, seq, userId, json) { Log.silly('HK_MESSAGE', json); let parsed; @@ -965,7 +964,7 @@ module.exports.create = function (cfg, cb) { var command = directMessageCommands[parsed[0]] || handleRPC; // run the command with the standard function signature - command(Server, seq, user, parsed); + command(Server, seq, userId, parsed); }; cfg.historyKeeper = { @@ -991,10 +990,10 @@ module.exports.create = function (cfg, cb) { channelName ]); }, - directMessage: function (Server, seq, user, json) { + directMessage: function (Server, seq, userId, json) { // netflux-server allows you to register an id with a handler // this handler is invoked every time someone sends a message to that id - onDirectMessage(Server, seq, user, json); + onDirectMessage(Server, seq, userId, json); }, }; diff --git a/lib/rpc.js b/lib/rpc.js index 47ea9976f..eaa4efbbf 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -287,7 +287,7 @@ var rpc = function (Env, Server, data, respond) { } if (isUnauthenticateMessage(msg)) { - return handleUnauthenticatedMessage(Env, msg, respond); + return handleUnauthenticatedMessage(Env, msg, respond, Server); } var signature = msg.shift(); From 3601bd6429e1eeaf610ef2e31daa5bad8defb4a3 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 3 Feb 2020 16:46:09 -0500 Subject: [PATCH 36/53] leave an XXX note to make sure we fix this typeError --- www/common/outer/messenger.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/www/common/outer/messenger.js b/www/common/outer/messenger.js index 4d5e42973..ed1eeadbb 100644 --- a/www/common/outer/messenger.js +++ b/www/common/outer/messenger.js @@ -893,6 +893,8 @@ define([ }; var clearOwnedChannel = function (ctx, id, cb) { + // XXX clients is undefined if you try to clear history from contacts + // while not using workers var channel = ctx.clients[id]; if (!channel) { return void cb({error: 'NO_CHANNEL'}); } if (!ctx.store.rpc) { return void cb({error: 'RPC_NOT_READY'}); } From 43307ffb1ab1c398a43c16bbdea5f327f9fb1658 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 3 Feb 2020 17:14:23 -0500 Subject: [PATCH 37/53] define all server intervals in a map so we can easily clear them all --- lib/api.js | 5 +++-- lib/commands/admin-rpc.js | 19 ++++++++++++++----- lib/historyKeeper.js | 1 - lib/rpc.js | 7 +++---- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/api.js b/lib/api.js index b66dcb4de..bbfefa4b8 100644 --- a/lib/api.js +++ b/lib/api.js @@ -13,6 +13,7 @@ module.exports.create = function (config) { config.store = _store; })); }).nThen(function (w) { + // XXX embed this in historyKeeper require("../storage/tasks").create(config, w(function (e, tasks) { if (e) { throw e; @@ -20,8 +21,8 @@ module.exports.create = function (config) { config.tasks = tasks; if (config.disableIntegratedTasks) { return; } - // XXX support stopping this interval - setInterval(function () { + config.intervals = config.intervals || {}; + config.intervals.taskExpiration = setInterval(function () { tasks.runAll(function (err) { if (err) { // either TASK_CONCURRENCY or an error with tasks.list diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 763bfb517..e3f0c6c58 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -15,11 +15,20 @@ var getActiveSessions = function (Env, Server, cb) { }; var shutdown = function (Env, Server, cb) { - return void cb('E_NOT_IMPLEMENTED'); - //clearInterval(Env.sessionExpirationInterval); - // XXX set a flag to prevent incoming database writes - // XXX disconnect all users and reject new connections - // XXX wait until all pending writes are complete + if (true) { + return void cb('E_NOT_IMPLEMENTED'); + } + + // disconnect all users and reject new connections + Server.shutdown(); + + // stop all intervals that may be running + Object.keys(Env.intervals).forEach(function (name) { + clearInterval(Env.intervals[name]); + }); + + // set a flag to prevent incoming database writes + // wait until all pending writes are complete // then process.exit(0); // and allow system functionality to restart the server }; diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index d5202876d..a1c09f9aa 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -359,7 +359,6 @@ module.exports.create = function (cfg, cb) { // close the channel store.closeChannel(channel, function () { - // XXX make sure that clients actually disconnect when we broadcast an error Server.channelBroadcast(channel, { error: 'EEXPIRED', channel: channel diff --git a/lib/rpc.js b/lib/rpc.js index eaa4efbbf..0df97efe3 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -378,6 +378,7 @@ RPC.create = function (config, cb) { var Env = { historyKeeper: config.historyKeeper, + intervals: config.intervals || {}, defaultStorageLimit: config.defaultStorageLimit, maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024), Sessions: {}, @@ -388,7 +389,6 @@ RPC.create = function (config, cb) { evPinnedPadsReady: mkEvent(true), limits: {}, admins: [], - sessionExpirationInterval: undefined, Log: Log, WARN: WARN, flushCache: config.flushCache, @@ -427,7 +427,7 @@ RPC.create = function (config, cb) { }; Quota.applyCustomLimits(Env); updateLimitDaily(); - setInterval(updateLimitDaily, 24*3600*1000); + Env.intervals.dailyLimitUpdate = setInterval(updateLimitDaily, 24*3600*1000); Pinning.loadChannelPins(Env); @@ -458,8 +458,7 @@ RPC.create = function (config, cb) { } }); // expire old sessions once per minute - // XXX allow for graceful shutdown - Env.sessionExpirationInterval = setInterval(function () { + Env.intervals.sessionExpirationInterval = setInterval(function () { Core.expireSessions(Sessions); }, Core.SESSION_EXPIRATION_TIME); }); From 65ba85d97b4ea8a2ca02d64e1462e3564fd58b9a Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 3 Feb 2020 17:14:42 -0500 Subject: [PATCH 38/53] clear historyKeeper cache when we trim a channel --- lib/commands/channel.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/commands/channel.js b/lib/commands/channel.js index ec4bf3fe6..872a2c445 100644 --- a/lib/commands/channel.js +++ b/lib/commands/channel.js @@ -105,9 +105,8 @@ Channel.removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, c }).nThen(function () { Env.msgStore.trimChannel(channelId, hash, function (err) { if (err) { return void cb(err); } - - - // XXX you must also clear the channel's index from historyKeeper cache + // clear historyKeeper's cache for this channel + Env.historyKeeper.channelClose(channelId); }); }); }; From d1c6e67d17493844ffe13b8326e532e9e7eb28fc Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 3 Feb 2020 18:30:29 -0500 Subject: [PATCH 39/53] throw if you try to mkAsync a non-function --- www/common/common-util.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/www/common/common-util.js b/www/common/common-util.js index da83b5960..5f6841983 100644 --- a/www/common/common-util.js +++ b/www/common/common-util.js @@ -34,6 +34,9 @@ }; Util.mkAsync = function (f) { + if (typeof(f) !== 'function') { + throw new Error('EXPECTED_FUNCTION'); + } return function () { var args = Array.prototype.slice.call(arguments); setTimeout(function () { From 88be40ede30e3f47bf0d2f262ac84b0820c15659 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 3 Feb 2020 18:32:21 -0500 Subject: [PATCH 40/53] standardize some function signatures and factor out a lot of boilerplate --- lib/commands/channel.js | 18 +++-- lib/commands/core.js | 17 +++-- lib/commands/metadata.js | 5 +- lib/commands/upload.js | 24 +++++- lib/rpc.js | 160 ++++++++++++--------------------------- 5 files changed, 95 insertions(+), 129 deletions(-) diff --git a/lib/commands/channel.js b/lib/commands/channel.js index 872a2c445..f50894016 100644 --- a/lib/commands/channel.js +++ b/lib/commands/channel.js @@ -6,10 +6,11 @@ const nThen = require("nthen"); const Core = require("./core"); const Metadata = require("./metadata"); -Channel.clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { +Channel.clearOwnedChannel = function (Env, safeKey, channelId, cb) { if (typeof(channelId) !== 'string' || channelId.length !== 32) { return cb('INVALID_ARGUMENTS'); } + var unsafeKey = Util.unescapeKeyCharacters(safeKey); Metadata.getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } @@ -24,13 +25,14 @@ Channel.clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { }); }; -Channel.removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { +Channel.removeOwnedChannel = function (Env, safeKey, channelId, cb) { if (typeof(channelId) !== 'string' || !Core.isValidId(channelId)) { return cb('INVALID_ARGUMENTS'); } + var unsafeKey = Util.unescapeKeyCharacters(safeKey); if (Env.blobStore.isFileId(channelId)) { - var safeKey = Util.escapeKeyCharacters(unsafeKey); + //var safeKey = Util.escapeKeyCharacters(unsafeKey); var blobId = channelId; return void nThen(function (w) { @@ -65,7 +67,7 @@ Channel.removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { if (err) { return void cb("E_PROOF_REMOVAL"); } - cb(); + cb(void 0, 'OK'); }); }); } @@ -83,12 +85,15 @@ Channel.removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { channelId: channelId, status: e? String(e): 'SUCCESS', }); - cb(e); + if (e) { + return void cb(e); + } + cb(void 0, 'OK'); }); }); }; -Channel.removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { +Channel.removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { // XXX UNSAFE nThen(function (w) { Metadata.getMetadata(Env, channelId, w(function (err, metadata) { if (err) { return void cb(err); } @@ -107,6 +112,7 @@ Channel.removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, c if (err) { return void cb(err); } // clear historyKeeper's cache for this channel Env.historyKeeper.channelClose(channelId); + cb(void 0, 'OK'); }); }); }; diff --git a/lib/commands/core.js b/lib/commands/core.js index 0bf7ff542..6c87959ad 100644 --- a/lib/commands/core.js +++ b/lib/commands/core.js @@ -59,13 +59,16 @@ Core.getSession = function (Sessions, key) { return user; }; -Core.expireSession = function (Sessions, key) { - var session = Sessions[key]; - if (!session) { return; } - if (session.blobstage) { - session.blobstage.close(); - } - delete Sessions[key]; +Core.expireSession = function (Sessions, key, cb) { + setTimeout(function () { + var session = Sessions[key]; + if (!session) { return; } + if (session.blobstage) { + session.blobstage.close(); + } + delete Sessions[key]; + cb(void 0, 'OK'); + }); }; var isTooOld = function (time, now) { diff --git a/lib/commands/metadata.js b/lib/commands/metadata.js index 3a21aae0b..fe05aa40d 100644 --- a/lib/commands/metadata.js +++ b/lib/commands/metadata.js @@ -5,6 +5,7 @@ const Meta = require("../metadata"); const BatchRead = require("../batch-read"); const WriteQueue = require("../write-queue"); const Core = require("./core"); +const Util = require("../common-util"); const batchMetadata = BatchRead("GET_METADATA"); Data.getMetadata = function (Env, channel, cb) { @@ -34,7 +35,9 @@ Data.getMetadata = function (Env, channel, cb) { } */ var queueMetadata = WriteQueue(); -Data.setMetadata = function (Env, data, unsafeKey, cb) { +Data.setMetadata = function (Env, safeKey, data, cb) { + var unsafeKey = Util.unescapeKeyCharacters(safeKey); + var channel = data.channel; var command = data.command; if (!channel || !Core.isValidId(channel)) { return void cb ('INVALID_CHAN'); } diff --git a/lib/commands/upload.js b/lib/commands/upload.js index f89cf2904..66868a65d 100644 --- a/lib/commands/upload.js +++ b/lib/commands/upload.js @@ -3,9 +3,9 @@ const Upload = module.exports; const Util = require("../common-util"); const Pinning = require("./pin-rpc"); const nThen = require("nthen"); +const Core = require("./core"); -// upload_status -Upload.upload_status = function (Env, safeKey, filesize, _cb) { // FIXME FILES +Upload.status = function (Env, safeKey, filesize, _cb) { // FIXME FILES var cb = Util.once(Util.mkAsync(_cb)); // validate that the provided size is actually a positive number @@ -29,9 +29,29 @@ Upload.upload_status = function (Env, safeKey, filesize, _cb) { // FIXME FILES Pinning.getFreeSpace(Env, safeKey, function (e, free) { if (e) { return void cb(e); } if (filesize >= free) { return cb('NOT_ENOUGH_SPACE'); } + + var user = Core.getSession(Env.Sessions, safeKey); + user.pendingUploadSize = filesize; + user.currentUploadSize = 0; + cb(void 0, false); }); }); }; +Upload.upload = function (Env, safeKey, chunk, cb) { + Env.blobStore.upload(safeKey, chunk, cb); +}; + +Upload.complete = function (Env, safeKey, arg, cb) { + Env.blobStore.complete(safeKey, arg, cb); +}; + +Upload.cancel = function (Env, safeKey, arg, cb) { + Env.blobStore.cancel(safeKey, arg, cb); +}; + +Upload.complete_owned = function (Env, safeKey, arg, cb) { + Env.blobStore.completeOwned(safeKey, arg, cb); +}; diff --git a/lib/rpc.js b/lib/rpc.js index 0df97efe3..1acac878c 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -112,6 +112,30 @@ var handleUnauthenticatedMessage = function (Env, msg, respond, Server) { } }; +const AUTHENTICATED_USER_TARGETED = { + RESET: Pinning.resetUserPins, + PIN: Pinning.pinChannel, + UNPIN: Pinning.unpinChannel, + CLEAR_OWNED_CHANNEL: Channel.clearOwnedChannel, + REMOVE_OWNED_CHANNEL: Channel.removeOwnedChannel, + UPLOAD_STATUS: Upload.status, + UPLOAD: Upload.upload, + UPLOAD_COMPLETE: Upload.complete, + UPLOAD_CANCEL: Upload.cancel, + OWNED_UPLOAD_COMPLETE: Upload.complete_owned, +}; + +const AUTHENTICATED_USER_SCOPED = { + GET_HASH: Pinning.getHash, + GET_TOTAL_SIZE: Pinning.getTotalSize, + UPDATE_LIMITS: Quota.updateLimits, + GET_LIMIT: Pinning.getLimit, + EXPIRE_SESSION: Core.expireSession, + REMOVE_PINS: Pinning.removePins, + TRIM_PINS: Pinning.trimPins, + SET_METADATA: Metadata.setMetadata, +}; + var handleAuthenticatedMessage = function (Env, map) { var msg = map.msg; var safeKey = map.safeKey; @@ -119,118 +143,34 @@ var handleAuthenticatedMessage = function (Env, map) { var Respond = map.Respond; var Server = map.Server; - Env.Log.silly('LOG_RPC', msg[0]); + var TYPE = msg[0]; + + Env.Log.silly('LOG_RPC', TYPE); + + if (typeof(AUTHENTICATED_USER_TARGETED[TYPE]) === 'function') { + return void AUTHENTICATED_USER_TARGETED[TYPE](Env, safeKey, msg[1], function (e, value) { + Env.WARN(e, value); + return void Respond(e, value); + }); + } + + if (typeof(AUTHENTICATED_USER_SCOPED[TYPE]) === 'function') { + return void AUTHENTICATED_USER_SCOPED[TYPE](Env, safeKey, function (e, value) { + if (e) { + Env.WARN(e, safeKey); + return void Respond(e); + } + Respond(e, value); + }); + } + switch (msg[0]) { case 'COOKIE': return void Respond(void 0); - case 'RESET': - return Pinning.resetUserPins(Env, safeKey, msg[1], function (e, hash) { // XXX USER_TARGETED - //WARN(e, hash); - return void Respond(e, hash); - }); - case 'PIN': - return Pinning.pinChannel(Env, safeKey, msg[1], function (e, hash) { // XXX USER_TARGETED - Env.WARN(e, hash); - Respond(e, hash); - }); - case 'UNPIN': - return Pinning.unpinChannel(Env, safeKey, msg[1], function (e, hash) { // XXX USER_TARGETED - Env.WARN(e, hash); - Respond(e, hash); - }); - case 'GET_HASH': - return void Pinning.getHash(Env, safeKey, function (e, hash) { // XXX USER_SCOPED - Env.WARN(e, hash); - Respond(e, hash); - }); - case 'GET_TOTAL_SIZE': // TODO cache this, since it will get called quite a bit - return Pinning.getTotalSize(Env, safeKey, function (e, size) { // XXX USER_SCOPED - if (e) { - Env.WARN(e, safeKey); - return void Respond(e); - } - Respond(e, size); - }); - case 'UPDATE_LIMITS': - return void Quota.updateLimits(Env, safeKey, function (e, limit) { // XXX USER_SCOPED - if (e) { - Env.WARN(e, limit); - return void Respond(e); - } - Respond(void 0, limit); - }); - case 'GET_LIMIT': - return void Pinning.getLimit(Env, safeKey, function (e, limit) { // XXX USER_SCOPED - if (e) { - Env.WARN(e, limit); - return void Respond(e); - } - Respond(void 0, limit); - }); - case 'EXPIRE_SESSION': - return void setTimeout(function () { // XXX USER_SCOPED - Core.expireSession(Env.Sessions, safeKey); - Respond(void 0, "OK"); - }); - case 'CLEAR_OWNED_CHANNEL': - return void Channel.clearOwnedChannel(Env, msg[1], publicKey, function (e, response) { // XXX USER_TARGETD_INVERSE - if (e) { return void Respond(e); } - Respond(void 0, response); - }); - - case 'REMOVE_OWNED_CHANNEL': - return void Channel.removeOwnedChannel(Env, msg[1], publicKey, function (e) { // XXX USER_TARGETED_INVERSE - if (e) { return void Respond(e); } - Respond(void 0, "OK"); - }); case 'TRIM_OWNED_CHANNEL_HISTORY': return void Channel.removeOwnedChannelHistory(Env, msg[1], publicKey, msg[2], function (e) { // XXX USER_TARGETED_DOUBLE if (e) { return void Respond(e); } Respond(void 0, 'OK'); }); - case 'REMOVE_PINS': - return void Pinning.removePins(Env, safeKey, function (e) { // XXX USER_SCOPED - if (e) { return void Respond(e); } - Respond(void 0, "OK"); - }); - case 'TRIM_PINS': - return void Pinning.trimPins(Env, safeKey, function (e) { // XXX USER_SCOPED - if (e) { return void Respond(e); } - Respond(void 0, "OK"); - }); - case 'UPLOAD': - return void Env.blobStore.upload(safeKey, msg[1], function (e, len) { // XXX USER_SCOPED_SPECIAL - Env.WARN(e, len); - Respond(e, len); - }); - case 'UPLOAD_STATUS': - var filesize = msg[1]; - return void Upload.upload_status(Env, safeKey, filesize, function (e, yes) { // XXX USER_TARGETED - if (!e && !yes) { - // no pending uploads, set the new size - var user = Core.getSession(Env.Sessions, safeKey); - user.pendingUploadSize = filesize; - user.currentUploadSize = 0; - } - Respond(e, yes); - }); - case 'UPLOAD_COMPLETE': - return void Env.blobStore.complete(safeKey, msg[1], function (e, hash) { // XXX USER_SCOPED_SPECIAL - Env.WARN(e, hash); - Respond(e, hash); - }); - case 'OWNED_UPLOAD_COMPLETE': - return void Env.blobStore.completeOwned(safeKey, msg[1], function (e, blobId) { // XXX USER_SCOPED_SPECIAL - Env.WARN(e, blobId); - Respond(e, blobId); - }); - case 'UPLOAD_CANCEL': - // msg[1] is fileSize - // if we pass it here, we can start an upload right away without calling - // UPLOAD_STATUS again - return void Env.blobStore.cancel(safeKey, msg[1], function (e) { // XXX USER_SCOPED_SPECIAL - Env.WARN(e, 'UPLOAD_CANCEL'); - Respond(e); - }); case 'WRITE_LOGIN_BLOCK': return void Block.writeLoginBlock(Env, msg[1], function (e) { // XXX SPECIAL if (e) { @@ -255,15 +195,9 @@ var handleAuthenticatedMessage = function (Env, map) { } Respond(void 0, result); }); - case 'SET_METADATA': - return void Metadata.setMetadata(Env, msg[1], publicKey, function (e, data) { // XXX USER_TARGETED_INVERSE - if (e) { - Env.WARN(e, data); - return void Respond(e); - } - Respond(void 0, data); - }); default: + console.log(msg); + throw new Error("OOPS"); return void Respond('UNSUPPORTED_RPC_CALL', msg); } }; From 46dfa026f0a03b6a234f5f4798cda61e1a603e77 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 3 Feb 2020 18:47:18 -0500 Subject: [PATCH 41/53] fix an API change that caused a typeError --- lib/commands/core.js | 24 ++++++++++++++---------- lib/rpc.js | 2 +- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/commands/core.js b/lib/commands/core.js index 6c87959ad..3e98d10dd 100644 --- a/lib/commands/core.js +++ b/lib/commands/core.js @@ -59,14 +59,18 @@ Core.getSession = function (Sessions, key) { return user; }; -Core.expireSession = function (Sessions, key, cb) { +Core.expireSession = function (Sessions, safeKey) { + var session = Sessions[safeKey]; + if (!session) { return; } + if (session.blobstage) { + session.blobstage.close(); + } + delete Sessions[safeKey]; +}; + +Core.expireSessionAsync = function (Env, safeKey, cb) { setTimeout(function () { - var session = Sessions[key]; - if (!session) { return; } - if (session.blobstage) { - session.blobstage.close(); - } - delete Sessions[key]; + Core.expireSession(Sessions, safeKey); cb(void 0, 'OK'); }); }; @@ -77,10 +81,10 @@ var isTooOld = function (time, now) { Core.expireSessions = function (Sessions) { var now = +new Date(); - Object.keys(Sessions).forEach(function (key) { - var session = Sessions[key]; + Object.keys(Sessions).forEach(function (safeKey) { + var session = Sessions[safeKey]; if (session && isTooOld(session.atime, now)) { - Core.expireSession(Sessions, key); + Core.expireSession(Sessions, safeKey); } }); }; diff --git a/lib/rpc.js b/lib/rpc.js index 1acac878c..d541a2550 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -130,7 +130,7 @@ const AUTHENTICATED_USER_SCOPED = { GET_TOTAL_SIZE: Pinning.getTotalSize, UPDATE_LIMITS: Quota.updateLimits, GET_LIMIT: Pinning.getLimit, - EXPIRE_SESSION: Core.expireSession, + EXPIRE_SESSION: Core.expireSessionAsync, REMOVE_PINS: Pinning.removePins, TRIM_PINS: Pinning.trimPins, SET_METADATA: Metadata.setMetadata, From a9f84021108fb4eeaad929bd1dfaca3ebb8fbda1 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 4 Feb 2020 11:04:52 +0100 Subject: [PATCH 42/53] Fix clear history in contacts --- www/common/outer/messenger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/common/outer/messenger.js b/www/common/outer/messenger.js index 4d5e42973..6587d9e4d 100644 --- a/www/common/outer/messenger.js +++ b/www/common/outer/messenger.js @@ -893,7 +893,7 @@ define([ }; var clearOwnedChannel = function (ctx, id, cb) { - var channel = ctx.clients[id]; + var channel = ctx.channels[id]; if (!channel) { return void cb({error: 'NO_CHANNEL'}); } if (!ctx.store.rpc) { return void cb({error: 'RPC_NOT_READY'}); } ctx.store.rpc.clearOwnedChannel(id, function (err) { From d065a3d11685ccb63b47fb02d415645a2f28ea1d Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 4 Feb 2020 15:19:12 +0100 Subject: [PATCH 43/53] Add transparency --- customize.dist/src/less2/include/corner.less | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/customize.dist/src/less2/include/corner.less b/customize.dist/src/less2/include/corner.less index d267f3683..6cf365c2e 100644 --- a/customize.dist/src/less2/include/corner.less +++ b/customize.dist/src/less2/include/corner.less @@ -31,7 +31,7 @@ bottom: 10px; width: 350px; padding: 10px; - background-color: @corner-blue; + background-color: fade(@corner-blue, 95%); border: 1px solid @corner-blue; color: @corner-white; z-index: 9999; @@ -40,7 +40,7 @@ //box-shadow: 0 0 10px 0 @corner-blue; &.cp-corner-alt { - background-color: @corner-white; + background-color: fade(@corner-white, 95%); border: 1px solid @corner-blue; color: @corner-blue; } From f8f3a48e8bbc1b3d1ee7407a8af7a04d43df9757 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 4 Feb 2020 11:15:44 -0500 Subject: [PATCH 44/53] use latest chainpad-server --- lib/historyKeeper.js | 2 +- package-lock.json | 9 ++++----- package.json | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index a1c09f9aa..b0f06e2fe 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -747,7 +747,7 @@ module.exports.create = function (cfg, cb) { // TODO compute lastKnownHash in a manner such that it will always skip past the metadata line? getHistoryAsync(channelName, lastKnownHash, false, (msg, readMore) => { - if (!msg) { return; } + if (!msg) { return; } // XXX msgCount++; // avoid sending the metadata message a second time if (isMetadataMessage(msg) && metadata_cache[channelName]) { return readMore(); } diff --git a/package-lock.json b/package-lock.json index eb4668a33..549bfbe97 100644 --- a/package-lock.json +++ b/package-lock.json @@ -113,14 +113,13 @@ } }, "chainpad-server": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/chainpad-server/-/chainpad-server-3.0.5.tgz", - "integrity": "sha512-USKOMSHsNjnme81Qy3nQ+ji9eCkBPokYH4T82LVHAI0aayTSCXcTPUDLVGDBCRqe8NsXU4io1WPXn1KiZwB8fA==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/chainpad-server/-/chainpad-server-4.0.0.tgz", + "integrity": "sha512-HWTdtyzBL/21asK21tf7fivfJm9MDhPz5odq39Az1ibRwZK4Eu+pDcKEHRKEW9KtugQxnFM8NwZoyW9DxAlA8w==", "requires": { - "nthen": "^0.1.8", + "nthen": "0.1.8", "pull-stream": "^3.6.9", "stream-to-pull-stream": "^1.7.3", - "tweetnacl": "~0.12.2", "ws": "^3.3.1" } }, diff --git a/package.json b/package.json index 057c823d9..ddd98c911 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ }, "dependencies": { "chainpad-crypto": "^0.2.2", - "chainpad-server": "^3.0.5", + "chainpad-server": "^4.0.0", "express": "~4.16.0", "fs-extra": "^7.0.0", "get-folder-size": "^2.0.1", From 0d6962068733d338a6a337361cf1802cff175da1 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 4 Feb 2020 17:33:51 -0500 Subject: [PATCH 45/53] tweak the metadata line handler to handle an edge case in trim history --- lib/metadata.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/metadata.js b/lib/metadata.js index de40043af..677b2e5b5 100644 --- a/lib/metadata.js +++ b/lib/metadata.js @@ -211,12 +211,14 @@ Meta.createLineHandler = function (ref, errorHandler) { line: JSON.stringify(line), }); } + + // the case above is special, everything else should increment the index + var index = ref.index++; if (typeof(line) === 'undefined') { return; } if (Array.isArray(line)) { try { handleCommand(ref.meta, line); - ref.index++; } catch (err2) { errorHandler("METADATA_COMMAND_ERR", { error: err2.stack, @@ -226,8 +228,15 @@ Meta.createLineHandler = function (ref, errorHandler) { return; } - if (ref.index === 0 && typeof(line) === 'object') { - ref.index++; + // the first line of a channel is processed before the dedicated metadata log. + // it can contain a map, in which case it should be used as the initial state. + // it's possible that a trim-history command was interrupted, in which case + // this first message might exist in parallel with the more recent metadata log + // which will contain the computed state of the previous metadata log + // which has since been archived. + // Thus, accept both the first and second lines you process as valid initial state + // preferring the second if it exists + if (index < 2 && line typeof(line) === 'object') { // special case! ref.meta = line; return; @@ -235,7 +244,7 @@ Meta.createLineHandler = function (ref, errorHandler) { errorHandler("METADATA_HANDLER_WEIRDLINE", { line: line, - index: ref.index++, + index: index, }); }; }; From 5cb266838afd72f3849e2668e503ba949431b4f9 Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 5 Feb 2020 11:28:27 +0100 Subject: [PATCH 46/53] Fix syntax error --- lib/metadata.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/metadata.js b/lib/metadata.js index 677b2e5b5..2b3a0b737 100644 --- a/lib/metadata.js +++ b/lib/metadata.js @@ -236,7 +236,7 @@ Meta.createLineHandler = function (ref, errorHandler) { // which has since been archived. // Thus, accept both the first and second lines you process as valid initial state // preferring the second if it exists - if (index < 2 && line typeof(line) === 'object') { + if (index < 2 && line && typeof(line) === 'object') { // special case! ref.meta = line; return; From 653d58433e116a8a104a1e3873806bc817957c88 Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 5 Feb 2020 12:16:04 +0100 Subject: [PATCH 47/53] Add link to profile in notifications --- .../src/less2/include/notifications.less | 9 +++++++ www/common/sframe-common-mailbox.js | 24 +++++++++++++++++-- www/notifications/app-notifications.less | 9 +++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/customize.dist/src/less2/include/notifications.less b/customize.dist/src/less2/include/notifications.less index 1e4430db2..a24ad32d3 100644 --- a/customize.dist/src/less2/include/notifications.less +++ b/customize.dist/src/less2/include/notifications.less @@ -8,6 +8,7 @@ @notif-height: 50px; .cp-notifications-container { max-width: 300px; + width: 300px; display: flex; flex-flow: column; & hr { @@ -16,6 +17,14 @@ .cp-notification { min-height: @notif-height; display: flex; + .cp-avatar { + .avatar_main(30px); + padding: 0 5px; + cursor: pointer; + &:hover { + background-color: rgba(0,0,0,0.1); + } + } .cp-notification-content { flex: 1; align-items: stretch; diff --git a/www/common/sframe-common-mailbox.js b/www/common/sframe-common-mailbox.js index 35bf5df70..c10a45ff3 100644 --- a/www/common/sframe-common-mailbox.js +++ b/www/common/sframe-common-mailbox.js @@ -1,12 +1,13 @@ define([ 'jquery', '/common/common-util.js', + '/common/common-hash.js', '/common/common-interface.js', '/common/common-ui-elements.js', '/common/notifications.js', '/common/hyperscript.js', '/customize/messages.js', -], function ($, Util, UI, UIElements, Notifications, h, Messages) { +], function ($, Util, Hash, UI, UIElements, Notifications, h, Messages) { var Mailbox = {}; Mailbox.create = function (Common) { @@ -53,9 +54,28 @@ define([ }; var createElement = mailbox.createElement = function (data) { var notif; + var avatar; + var type = Util.find(data, ['content', 'msg', 'type']); + var userData = ['FRIEND_REQUEST'].indexOf(type) !== -1 ? Util.find(data, ['content', 'msg', 'content']) + : Util.find(data, ['content', 'msg', 'content', 'user']); + if (userData && typeof(userData) === "object" && userData.profile) { + avatar = h('span.cp-avatar'); + Common.displayAvatar($(avatar), userData.avatar, userData.displayName || userData.name); + $(avatar).click(function (e) { + e.stopPropagation(); + Common.openURL(Hash.hashToHref(userData.profile, 'profile')); + }); + } else { + console.warn(data); + + } notif = h('div.cp-notification', { 'data-hash': data.content.hash - }, [h('div.cp-notification-content', h('p', formatData(data)))]); + }, [ + avatar, + h('div.cp-notification-content', + h('p', formatData(data))) + ]); if (typeof(data.content.getFormatText) === "function") { $(notif).find('.cp-notification-content p').html(data.content.getFormatText()); diff --git a/www/notifications/app-notifications.less b/www/notifications/app-notifications.less index dfc1da7f7..5199ae5b6 100644 --- a/www/notifications/app-notifications.less +++ b/www/notifications/app-notifications.less @@ -1,5 +1,6 @@ @import (reference) '../../customize/src/less2/include/framework.less'; @import (reference) '../../customize/src/less2/include/sidebar-layout.less'; +@import (reference) '../../customize/src/less2/include/avatar.less'; &.cp-app-notifications { @@ -86,6 +87,14 @@ display: block; } } + .cp-avatar { + .avatar_main(48px); + padding: 0 10px; + cursor: pointer; + &:hover { + background-color: rgba(0,0,0,0.1); + } + } &.cp-app-notification-archived { background-color: #f1f1f1; } From 2ee38ccc42af2a626c5661938041015823a80cae Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 5 Feb 2020 13:33:32 +0100 Subject: [PATCH 48/53] lint compliance --- www/common/common-messaging.js | 12 ++- www/common/common-ui-elements.js | 57 ++------------ www/common/notifications.js | 18 ++++- www/common/outer/async-store.js | 32 +++----- www/common/outer/mailbox-handlers.js | 114 ++++++++++++++++----------- www/common/outer/mailbox.js | 14 +++- www/common/outer/team.js | 14 +--- www/common/sframe-common-mailbox.js | 7 +- www/pad/inner.js | 4 +- 9 files changed, 126 insertions(+), 146 deletions(-) diff --git a/www/common/common-messaging.js b/www/common/common-messaging.js index feb3d79d3..15d0408f7 100644 --- a/www/common/common-messaging.js +++ b/www/common/common-messaging.js @@ -53,10 +53,18 @@ define([ return list; }; + Msg.declineFriendRequest = function (store, data, cb) { + store.mailbox.sendTo('DECLINE_FRIEND_REQUEST', {}, { + channel: data.notifications, + curvePublic: data.curvePublic + }, function (obj) { + cb(obj); + }); + }; Msg.acceptFriendRequest = function (store, data, cb) { var friend = getFriend(store.proxy, data.curvePublic) || {}; var myData = createData(store.proxy, friend.channel || data.channel); - store.mailbox.sendTo('ACCEPT_FRIEND_REQUEST', myData, { + store.mailbox.sendTo('ACCEPT_FRIEND_REQUEST', { user: myData }, { channel: data.notifications, curvePublic: data.curvePublic }, function (obj) { @@ -110,7 +118,7 @@ define([ var proxy = store.proxy; var friend = proxy.friends[curvePublic]; if (!friend) { return void cb({error: 'ENOENT'}); } - if (!friend.notifications || !friend.channel) { return void cb({error: 'EINVAL'}); } + if (!friend.notifications) { return void cb({error: 'EINVAL'}); } store.mailbox.sendTo('UNFRIEND', { curvePublic: proxy.curvePublic diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index 8bfa02e3a..c60ef652d 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -212,15 +212,7 @@ define([ common.mailbox.sendTo("RM_OWNER", { channel: channel, title: data.title, - pending: pending, - user: { - displayName: user.name, - avatar: user.avatar, - profile: user.profile, - notifications: user.notifications, - curvePublic: user.curvePublic, - edPublic: priv.edPublic - } + pending: pending }, { channel: friend.notifications, curvePublic: friend.curvePublic @@ -363,15 +355,7 @@ define([ channel: channel, href: data.href, password: data.password, - title: data.title, - user: { - displayName: user.name, - avatar: user.avatar, - profile: user.profile, - notifications: user.notifications, - curvePublic: user.curvePublic, - edPublic: priv.edPublic - } + title: data.title }, { channel: friend.notifications, curvePublic: friend.curvePublic @@ -4335,7 +4319,8 @@ define([ UIElements.displayFriendRequestModal = function (common, data) { var msg = data.content.msg; - var text = Messages._getKey('contacts_request', [Util.fixHTML(msg.content.displayName)]); + var userData = msg.content.user; + var text = Messages._getKey('contacts_request', [Util.fixHTML(userData.displayName)]); var todo = function (yes) { common.getSframeChannel().query("Q_ANSWER_FRIEND_REQUEST", { @@ -4362,7 +4347,6 @@ define([ UIElements.displayAddOwnerModal = function (common, data) { var priv = common.getMetadataMgr().getPrivateData(); - var user = common.getMetadataMgr().getUserData(); var sframeChan = common.getSframeChannel(); var msg = data.content.msg; @@ -4397,15 +4381,7 @@ define([ href: msg.content.href, password: msg.content.password, title: msg.content.title, - answer: yes, - user: { - displayName: user.name, - avatar: user.avatar, - profile: user.profile, - notifications: user.notifications, - curvePublic: user.curvePublic, - edPublic: priv.edPublic - } + answer: yes }, { channel: msg.content.user.notifications, curvePublic: msg.content.user.curvePublic @@ -4486,7 +4462,6 @@ define([ }; UIElements.displayAddTeamOwnerModal = function (common, data) { var priv = common.getMetadataMgr().getPrivateData(); - var user = common.getMetadataMgr().getUserData(); var sframeChan = common.getSframeChannel(); var msg = data.content.msg; @@ -4503,15 +4478,7 @@ define([ common.mailbox.sendTo("ADD_OWNER_ANSWER", { teamChannel: msg.content.teamChannel, title: msg.content.title, - answer: yes, - user: { - displayName: user.name, - avatar: user.avatar, - profile: user.profile, - notifications: user.notifications, - curvePublic: user.curvePublic, - edPublic: priv.edPublic - } + answer: yes }, { channel: msg.content.user.notifications, curvePublic: msg.content.user.curvePublic @@ -4627,8 +4594,6 @@ define([ }; UIElements.displayInviteTeamModal = function (common, data) { - var priv = common.getMetadataMgr().getPrivateData(); - var user = common.getMetadataMgr().getUserData(); var msg = data.content.msg; var name = Util.fixHTML(msg.content.user.displayName) || Messages.anonymous; @@ -4649,15 +4614,7 @@ define([ common.mailbox.sendTo("INVITE_TO_TEAM_ANSWER", { answer: yes, teamChannel: msg.content.team.channel, - teamName: teamName, - user: { - displayName: user.name, - avatar: user.avatar, - profile: user.profile, - notifications: user.notifications, - curvePublic: user.curvePublic, - edPublic: priv.edPublic - } + teamName: teamName }, { channel: msg.content.user.notifications, curvePublic: msg.content.user.curvePublic diff --git a/www/common/notifications.js b/www/common/notifications.js index 192d2b4c6..8546f2787 100644 --- a/www/common/notifications.js +++ b/www/common/notifications.js @@ -29,7 +29,9 @@ define([ handlers['FRIEND_REQUEST'] = function (common, data) { var content = data.content; var msg = content.msg; - var name = Util.fixHTML(msg.content.displayName) || Messages.anonymous; + var userData = msg.content.user || msg.content; + var name = Util.fixHTML(userData.displayName) || Messages.anonymous; + msg.content = { user: userData }; // Display the notification content.getFormatText = function () { @@ -37,7 +39,7 @@ define([ }; // Check authenticity - if (msg.author !== msg.content.curvePublic) { return; } + if (msg.author !== userData.curvePublic) { return; } // if not archived, add handlers if (!content.archived) { @@ -51,7 +53,11 @@ define([ handlers['FRIEND_REQUEST_ACCEPTED'] = function (common, data) { var content = data.content; var msg = content.msg; - var name = Util.fixHTML(msg.content.name) || Messages.anonymous; + var userData = typeof(msg.content.user) === "object" ? msg.content.user : { + displayName: msg.content.name, + curvePublic: msg.content.user + }; + var name = Util.fixHTML(userData.displayName) || Messages.anonymous; content.getFormatText = function () { return Messages._getKey('friendRequest_accepted', [name]); }; @@ -63,7 +69,11 @@ define([ handlers['FRIEND_REQUEST_DECLINED'] = function (common, data) { var content = data.content; var msg = content.msg; - var name = Util.fixHTML(msg.content.name) || Messages.anonymous; + var userData = typeof(msg.content.user) === "object" ? msg.content.user : { + displayName: msg.content.name, + curvePublic: msg.content.user + }; + var name = Util.fixHTML(userData.displayName) || Messages.anonymous; content.getFormatText = function () { return Messages._getKey('friendRequest_declined', [name]); }; diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index 51c04066f..c544754ba 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -1267,15 +1267,15 @@ define([ // If we accept the request, add the friend to the list if (value) { - Messaging.acceptFriendRequest(store, msg.content, function (obj) { + Messaging.acceptFriendRequest(store, msg.content.user, function (obj) { if (obj && obj.error) { return void cb(obj); } Messaging.addToFriendList({ proxy: store.proxy, realtime: store.realtime, pinPads: function (data, cb) { Store.pinPads(null, data, cb); }, - }, msg.content, function (err) { + }, msg.content.user, function (err) { if (store.messenger) { - store.messenger.onFriendAdded(msg.content); + store.messenger.onFriendAdded(msg.content.user); } broadcast([], "UPDATE_METADATA"); if (err) { return void cb({error: err}); } @@ -1285,12 +1285,7 @@ define([ return; } // Otherwise, just remove the notification - store.mailbox.sendTo('DECLINE_FRIEND_REQUEST', { - displayName: store.proxy['cryptpad.username'] - }, { - channel: msg.content.notifications, - curvePublic: msg.content.curvePublic - }, function (obj) { + Messaging.declineFriendRequest(store, msg.content.user, function (obj) { broadcast([], "UPDATE_METADATA"); cb(obj); }); @@ -1312,8 +1307,9 @@ define([ store.proxy.friends_pending[data.curvePublic] = +new Date(); broadcast([], "UPDATE_METADATA"); - var myData = Messaging.createData(store.proxy); - store.mailbox.sendTo('FRIEND_REQUEST', myData, { + store.mailbox.sendTo('FRIEND_REQUEST', { + user: Messaging.createData(store.proxy) + }, { channel: data.notifications, curvePublic: data.curvePublic }, function (obj) { @@ -1649,11 +1645,8 @@ define([ // If send is true, send the request to the owner. if (owner) { if (data.send) { - var myData = Messaging.createData(store.proxy); - delete myData.channel; store.mailbox.sendTo('REQUEST_PAD_ACCESS', { - channel: data.channel, - user: myData + channel: data.channel }, { channel: owner.notifications, curvePublic: owner.curvePublic @@ -1687,13 +1680,10 @@ define([ } })) { return void cb({error: 'ENOTFOUND'}); } - var myData = Messaging.createData(store.proxy); - delete myData.channel; store.mailbox.sendTo("GIVE_PAD_ACCESS", { channel: channel, href: href, - title: title, - user: myData + title: title }, { channel: data.user.notifications, curvePublic: data.user.curvePublic @@ -1727,13 +1717,11 @@ define([ } // Tell all the owners that the pad was deleted from the server var curvePublic = store.proxy.curvePublic; - var myData = Messaging.createData(store.proxy, false); m.forEach(function (obj) { var mb = JSON.parse(obj); if (mb.curvePublic === curvePublic) { return; } store.mailbox.sendTo('OWNED_PAD_REMOVED', { - channel: channel, - user: myData + channel: channel }, { channel: mb.notifications, curvePublic: mb.curvePublic diff --git a/www/common/outer/mailbox-handlers.js b/www/common/outer/mailbox-handlers.js index 253157361..d66ff516e 100644 --- a/www/common/outer/mailbox-handlers.js +++ b/www/common/outer/mailbox-handlers.js @@ -4,6 +4,7 @@ define([ '/common/common-util.js', ], function (Messaging, Hash, Util) { + // Random timeout between 10 and 30 times your sync time (lag + chainpad sync) var getRandomTimeout = function (ctx) { var lag = ctx.store.realtime.getLag().lag || 0; return (Math.max(0, lag) + 300) * 20 * (0.5 + Math.random()); @@ -22,9 +23,11 @@ define([ // Store the friend request displayed to avoid duplicates var friendRequest = {}; handlers['FRIEND_REQUEST'] = function (ctx, box, data, cb) { + // Old format: data was stored directly in "content" + var userData = data.msg.content.user || data.msg.content; // Check if the request is valid (send by the correct user) - if (data.msg.author !== data.msg.content.curvePublic) { + if (data.msg.author !== userData.curvePublic) { return void cb(true); } @@ -40,7 +43,8 @@ define([ if (Messaging.getFriend(ctx.store.proxy, data.msg.author) || ctx.store.proxy.friends_pending[data.msg.author]) { delete ctx.store.proxy.friends_pending[data.msg.author]; - Messaging.acceptFriendRequest(ctx.store, data.msg.content, function (obj) { + + Messaging.acceptFriendRequest(ctx.store, userData, function (obj) { if (obj && obj.error) { return void cb(); } @@ -48,10 +52,10 @@ define([ proxy: ctx.store.proxy, realtime: ctx.store.realtime, pinPads: ctx.pinPads - }, data.msg.content, function (err) { - if (err) { console.error(err); } + }, userData, function (err) { + if (err) { return void console.error(err); } if (ctx.store.messenger) { - ctx.store.messenger.onFriendAdded(data.msg.content); + ctx.store.messenger.onFriendAdded(userData); } }); ctx.updateMetadata(); @@ -63,96 +67,110 @@ define([ cb(); }; removeHandlers['FRIEND_REQUEST'] = function (ctx, box, data) { - if (friendRequest[data.content.curvePublic]) { - delete friendRequest[data.content.curvePublic]; + var userData = data.content.user || data.content; + if (friendRequest[userData.curvePublic]) { + delete friendRequest[userData.curvePublic]; } }; + // The DECLINE and ACCEPT messages act on the contacts data + // They are processed with a random timeout to avoid having + // multiple workers trying to add or remove the contacts at + // the same time. Once processed, they are dismissed. + // We must dismiss them and send another message to our own + // mailbox for the UI part otherwise it would automatically + // accept or decline future requests from the same user + // until the message is manually dismissed. + var friendRequestDeclined = {}; handlers['DECLINE_FRIEND_REQUEST'] = function (ctx, box, data, cb) { + // Old format: data was stored directly in "content" + var userData = data.msg.content.user || data.msg.content; + if (!userData.curvePublic) { userData.curvePublic = data.msg.author; } + + // Our friend request was declined. setTimeout(function () { - // Our friend request was declined. - if (!ctx.store.proxy.friends_pending[data.msg.author]) { return; } + // Only dismissed once in the timeout to make sure we won't lose + // the data if we close the worker before adding the friend + cb(true); + // Make sure we really sent it + if (!ctx.store.proxy.friends_pending[data.msg.author]) { return; } // Remove the pending message and display the "declined" state in the UI delete ctx.store.proxy.friends_pending[data.msg.author]; + ctx.updateMetadata(); if (friendRequestDeclined[data.msg.author]) { return; } + friendRequestDeclined[data.msg.author] = true; box.sendMessage({ type: 'FRIEND_REQUEST_DECLINED', - content: { - user: data.msg.author, - name: data.msg.content.displayName - } - }, function () { - if (friendRequestDeclined[data.msg.author]) { - // TODO remove our message because another one was sent first? - } - friendRequestDeclined[data.msg.author] = true; - }); + content: { user: userData } + }, function () {}); }, getRandomTimeout(ctx)); - cb(true); }; + // UI for declined friend request handlers['FRIEND_REQUEST_DECLINED'] = function (ctx, box, data, cb) { ctx.updateMetadata(); - if (friendRequestDeclined[data.msg.content.user]) { return void cb(true); } - friendRequestDeclined[data.msg.content.user] = true; + var curve = data.msg.content.user.curvePublic || data.msg.content.user; + if (friendRequestDeclined[curve]) { return void cb(true); } + friendRequestDeclined[curve] = true; cb(); }; removeHandlers['FRIEND_REQUEST_DECLINED'] = function (ctx, box, data) { - if (friendRequestDeclined[data.content.user]) { - delete friendRequestDeclined[data.content.user]; - } + var curve = data.content.user.curvePublic || data.content.user; + if (friendRequestDeclined[curve]) { delete friendRequestDeclined[curve]; } }; var friendRequestAccepted = {}; handlers['ACCEPT_FRIEND_REQUEST'] = function (ctx, box, data, cb) { + // Old format: data was stored directly in "content" + var userData = data.msg.content.user || data.msg.content; + // Our friend request was accepted. setTimeout(function () { + // Only dismissed once in the timeout to make sure we won't lose + // the data if we close the worker before adding the friend + cb(true); + // Make sure we really sent it if (!ctx.store.proxy.friends_pending[data.msg.author]) { return; } + // Remove the pending state. It will also us to send a new request in case of error + delete ctx.store.proxy.friends_pending[data.msg.author]; + // And add the friend Messaging.addToFriendList({ proxy: ctx.store.proxy, realtime: ctx.store.realtime, pinPads: ctx.pinPads - }, data.msg.content, function (err) { - if (err) { console.error(err); } - delete ctx.store.proxy.friends_pending[data.msg.author]; - if (ctx.store.messenger) { - ctx.store.messenger.onFriendAdded(data.msg.content); - } + }, userData, function (err) { + if (err) { return void console.error(err); } + // Load the chat if contacts app loaded + if (ctx.store.messenger) { ctx.store.messenger.onFriendAdded(userData); } + // Update the userlist ctx.updateMetadata(); // If you have a profile page open, update it if (ctx.store.modules['profile']) { ctx.store.modules['profile'].update(); } - if (friendRequestAccepted[data.msg.author]) { return; } // Display the "accepted" state in the UI + if (friendRequestAccepted[data.msg.author]) { return; } + friendRequestAccepted[data.msg.author] = true; box.sendMessage({ type: 'FRIEND_REQUEST_ACCEPTED', - content: { - user: data.msg.author, - name: data.msg.content.displayName - } - }, function () { - if (friendRequestAccepted[data.msg.author]) { - // TODO remove our message because another one was sent first? - } - friendRequestAccepted[data.msg.author] = true; - }); + content: { user: userData } + }, function () {}); }); }, getRandomTimeout(ctx)); - cb(true); }; + // UI for accepted friend request handlers['FRIEND_REQUEST_ACCEPTED'] = function (ctx, box, data, cb) { ctx.updateMetadata(); - if (friendRequestAccepted[data.msg.content.user]) { return void cb(true); } - friendRequestAccepted[data.msg.content.user] = true; + var curve = data.msg.content.user.curvePublic || data.msg.content.user; + if (friendRequestAccepted[curve]) { return void cb(true); } + friendRequestAccepted[curve] = true; cb(); }; removeHandlers['FRIEND_REQUEST_ACCEPTED'] = function (ctx, box, data) { - if (friendRequestAccepted[data.content.user]) { - delete friendRequestAccepted[data.content.user]; - } + var curve = data.content.user.curvePublic || data.content.user; + if (friendRequestAccepted[curve]) { delete friendRequestAccepted[curve]; } }; handlers['UNFRIEND'] = function (ctx, box, data, cb) { diff --git a/www/common/outer/mailbox.js b/www/common/outer/mailbox.js index b84e98dd7..ce9bbe850 100644 --- a/www/common/outer/mailbox.js +++ b/www/common/outer/mailbox.js @@ -2,11 +2,12 @@ define([ '/common/common-util.js', '/common/common-hash.js', '/common/common-realtime.js', + '/common/common-messaging.js', '/common/notify.js', '/common/outer/mailbox-handlers.js', '/bower_components/chainpad-netflux/chainpad-netflux.js', '/bower_components/chainpad-crypto/crypto.js', -], function (Util, Hash, Realtime, Notify, Handlers, CpNetflux, Crypto) { +], function (Util, Hash, Realtime, Messaging, Notify, Handlers, CpNetflux, Crypto) { var Mailbox = {}; var TYPES = [ @@ -96,6 +97,12 @@ proxy.mailboxes = { var crypto = Crypto.Mailbox.createEncryptor(keys); + // Always send your data + if (typeof(msg) === "object" && !msg.user) { + var myData = Messaging.createData(ctx.store.proxy, false); + msg.user = myData; + } + var text = JSON.stringify({ type: type, content: msg @@ -187,6 +194,11 @@ proxy.mailboxes = { history: [], // All the hashes loaded from the server in corretc order content: {}, // Content of the messages that should be displayed sendMessage: function (msg) { // To send a message to our box + // Always send your data + if (typeof(msg) === "object" && !msg.user) { + var myData = Messaging.createData(ctx.store.proxy, false); + msg.user = myData; + } try { msg = JSON.stringify(msg); } catch (e) { diff --git a/www/common/outer/team.js b/www/common/outer/team.js index ce79a800f..24952f5c9 100644 --- a/www/common/outer/team.js +++ b/www/common/outer/team.js @@ -909,13 +909,11 @@ define([ })); }).nThen(function (waitFor) { // Send mailbox to offer ownership - var myData = Messaging.createData(ctx.store.proxy, false); ctx.store.mailbox.sendTo("ADD_OWNER", { teamChannel: teamData.channel, chatChannel: Util.find(teamData, ['keys', 'chat', 'channel']), rosterChannel: Util.find(teamData, ['keys', 'roster', 'channel']), - title: teamData.metadata.name, - user: myData + title: teamData.metadata.name }, { channel: user.notifications, curvePublic: user.curvePublic @@ -969,12 +967,10 @@ define([ })); }).nThen(function (waitFor) { // Send mailbox to offer ownership - var myData = Messaging.createData(ctx.store.proxy, false); ctx.store.mailbox.sendTo("RM_OWNER", { teamChannel: teamData.channel, title: teamData.metadata.name, - pending: isPendingOwner, - user: myData + pending: isPendingOwner }, { channel: user.notifications, curvePublic: user.curvePublic @@ -1104,11 +1100,9 @@ define([ if (!team) { return void cb ({error: 'ENOENT'}); } // Send mailbox to offer ownership - var myData = Messaging.createData(ctx.store.proxy, false); ctx.store.mailbox.sendTo("TEAM_EDIT_RIGHTS", { state: state, - teamData: getInviteData(ctx, teamId, state), - user: myData + teamData: getInviteData(ctx, teamId, state) }, { channel: user.notifications, curvePublic: user.curvePublic @@ -1175,7 +1169,6 @@ define([ team.roster.add(obj, function (err) { if (err && err !== 'NO_CHANGE') { return void cb({error: err}); } ctx.store.mailbox.sendTo('INVITE_TO_TEAM', { - user: Messaging.createData(ctx.store.proxy, false), team: getInviteData(ctx, teamId) }, { channel: user.notifications, @@ -1202,7 +1195,6 @@ define([ if (!userData || !userData.notifications) { return cb(); } ctx.store.mailbox.sendTo('KICKED_FROM_TEAM', { pending: data.pending, - user: Messaging.createData(ctx.store.proxy, false), teamChannel: getInviteData(ctx, teamId).channel, teamName: getInviteData(ctx, teamId).metadata.name }, { diff --git a/www/common/sframe-common-mailbox.js b/www/common/sframe-common-mailbox.js index c10a45ff3..a5602fc3f 100644 --- a/www/common/sframe-common-mailbox.js +++ b/www/common/sframe-common-mailbox.js @@ -55,9 +55,7 @@ define([ var createElement = mailbox.createElement = function (data) { var notif; var avatar; - var type = Util.find(data, ['content', 'msg', 'type']); - var userData = ['FRIEND_REQUEST'].indexOf(type) !== -1 ? Util.find(data, ['content', 'msg', 'content']) - : Util.find(data, ['content', 'msg', 'content', 'user']); + var userData = Util.find(data, ['content', 'msg', 'content', 'user']); if (userData && typeof(userData) === "object" && userData.profile) { avatar = h('span.cp-avatar'); Common.displayAvatar($(avatar), userData.avatar, userData.displayName || userData.name); @@ -65,9 +63,6 @@ define([ e.stopPropagation(); Common.openURL(Hash.hashToHref(userData.profile, 'profile')); }); - } else { - console.warn(data); - } notif = h('div.cp-notification', { 'data-hash': data.content.hash diff --git a/www/pad/inner.js b/www/pad/inner.js index 52ae7325a..3f6e717ce 100644 --- a/www/pad/inner.js +++ b/www/pad/inner.js @@ -736,9 +736,9 @@ define([ }); framework._.sfCommon.isPadStored(function (err, val) { - //if (!val) { return; } + if (!val) { return; } var b64images = $inner.find('img[src^="data:image"]:not(.cke_reset)'); - if (true || b64images.length && framework._.sfCommon.isLoggedIn()) { + if (b64images.length && framework._.sfCommon.isLoggedIn()) { var no = h('button.cp-corner-cancel', Messages.cancel); var yes = h('button.cp-corner-primary', Messages.ok); var actions = h('div', [no, yes]); From f94713cecaa4e39308c9e7885ece7fe7b3b87a1d Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 5 Feb 2020 08:05:35 -0500 Subject: [PATCH 49/53] update package-lock to use latest chainpad-server --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 549bfbe97..8be1c779c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -113,9 +113,9 @@ } }, "chainpad-server": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/chainpad-server/-/chainpad-server-4.0.0.tgz", - "integrity": "sha512-HWTdtyzBL/21asK21tf7fivfJm9MDhPz5odq39Az1ibRwZK4Eu+pDcKEHRKEW9KtugQxnFM8NwZoyW9DxAlA8w==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/chainpad-server/-/chainpad-server-4.0.1.tgz", + "integrity": "sha512-duV57hO0o2cKaOwwWdDeO3hgN2thAqoQENrjozhamGrUjF9bFiNW2cq3Dg3HjOY6yeMNIGgj0jMuLJjTSERKhQ==", "requires": { "nthen": "0.1.8", "pull-stream": "^3.6.9", From 443fb6e22dd45e0197d226b0117d46634a3fe5f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Benqu=C3=A9?= Date: Wed, 5 Feb 2020 13:11:25 +0000 Subject: [PATCH 50/53] use variable for password input height --- customize.dist/src/less2/include/buttons.less | 6 +++++- customize.dist/src/less2/include/sidebar-layout.less | 1 + customize.dist/src/less2/include/variables.less | 1 + www/settings/app-settings.less | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/customize.dist/src/less2/include/buttons.less b/customize.dist/src/less2/include/buttons.less index ad6aaf9cc..8d906de62 100644 --- a/customize.dist/src/less2/include/buttons.less +++ b/customize.dist/src/less2/include/buttons.less @@ -10,7 +10,7 @@ @alertify-input-fg: @colortheme_modal-input-fg; input:not(.form-control), textarea { - background-color: @alertify-input-fg; + // background-color: @alertify-input-fg; color: @cryptpad_text_col; border: 1px solid @alertify-input-bg; width: 100%; @@ -23,6 +23,10 @@ } } + input:not(.form-control) { + height: @variables_input-height; + } + div.cp-alertify-type { display: flex; input { diff --git a/customize.dist/src/less2/include/sidebar-layout.less b/customize.dist/src/less2/include/sidebar-layout.less index 1f9c92457..59ea22c76 100644 --- a/customize.dist/src/less2/include/sidebar-layout.less +++ b/customize.dist/src/less2/include/sidebar-layout.less @@ -117,6 +117,7 @@ //border-radius: 0 0.25em 0.25em 0; //border: 1px solid #adadad; border-left: 0px; + height: @variables_input-height; } } &>div { diff --git a/customize.dist/src/less2/include/variables.less b/customize.dist/src/less2/include/variables.less index ba6c642e2..570779f05 100644 --- a/customize.dist/src/less2/include/variables.less +++ b/customize.dist/src/less2/include/variables.less @@ -3,6 +3,7 @@ // Elements size @variables_bar-height: 32px; +@variables_input-height: 38px; // Used in modal.less and alertify.less @variables_padding: 12px; diff --git a/www/settings/app-settings.less b/www/settings/app-settings.less index 30ff869a0..b6cd22fc1 100644 --- a/www/settings/app-settings.less +++ b/www/settings/app-settings.less @@ -138,6 +138,7 @@ padding: 5px; padding-left: 15px; &[type="number"] { + height: @variables_input-height + 2px; // to avoid cropped numbers border-right: 1px solid #adadad; } &[type="checkbox"] { From 78795a3b4d247e24d524e63fb5e95d657262f18f Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 5 Feb 2020 14:54:23 +0100 Subject: [PATCH 51/53] Login or register in profile for anonymous users --- customize.dist/src/less2/include/corner.less | 4 +++- www/profile/inner.js | 23 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/customize.dist/src/less2/include/corner.less b/customize.dist/src/less2/include/corner.less index 6cf365c2e..feec62165 100644 --- a/customize.dist/src/less2/include/corner.less +++ b/customize.dist/src/less2/include/corner.less @@ -137,7 +137,9 @@ button { padding: 5px; color: @corner-white; - margin-left: 10px; + &:not(:first-child) { + margin-left: 10px; + } outline: none; text-transform: uppercase; border: 1px solid @corner-white; diff --git a/www/profile/inner.js b/www/profile/inner.js index 0f4a17746..773b21206 100644 --- a/www/profile/inner.js +++ b/www/profile/inner.js @@ -570,6 +570,29 @@ define([ return; } + if (!common.isLoggedIn()) { + var login = h('button.cp-corner-primary', Messages.login_login); + var register = h('button.cp-corner-primary', Messages.login_register); + var cancel = h('button.cp-corner-cancel', Messages.cancel); + var actions = h('div', [cancel, register, login]); + var modal = UI.cornerPopup(Messages.profile_login || "You need to log in to add this user to your contacts", actions, '', {alt: true}); // XXX + $(register).click(function () { + common.setLoginRedirect(function () { + common.gotoURL('/register/'); + }); + modal.delete(); + }); + $(login).click(function () { + common.setLoginRedirect(function () { + common.gotoURL('/login/'); + }); + modal.delete(); + }); + $(cancel).click(function () { + modal.delete(); + }); + } + var listmapConfig = { data: {}, common: common, From 768dc718037bdfd004d84f95575b361274f284bd Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 5 Feb 2020 15:35:19 +0100 Subject: [PATCH 52/53] Fix remote changes in codemirror hijacking cursor --- www/common/sframe-common-codemirror.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/www/common/sframe-common-codemirror.js b/www/common/sframe-common-codemirror.js index d16bea97a..8cb1b7c2e 100644 --- a/www/common/sframe-common-codemirror.js +++ b/www/common/sframe-common-codemirror.js @@ -62,15 +62,16 @@ define([ }); editor._noCursorUpdate = false; - editor.state.focused = true; + editor.scrollTo(scroll.left, scroll.top); + + if (!editor.state.focused) { return; } + if(selects[0] === selects[1]) { editor.setCursor(posToCursor(selects[0], remoteDoc)); } else { editor.setSelection(posToCursor(selects[0], remoteDoc), posToCursor(selects[1], remoteDoc)); } - - editor.scrollTo(scroll.left, scroll.top); }; module.getHeadingText = function (editor) { From 1c013d8a4f24b451c6e0aeda525a0b96cd672e8c Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 5 Feb 2020 16:18:09 +0100 Subject: [PATCH 53/53] Fix stacking disconnection alerts --- www/common/common-interface.js | 5 +++++ www/common/common-ui-elements.js | 15 +++++++++++++++ www/common/onlyoffice/inner.js | 11 ++--------- www/common/sframe-app-framework.js | 4 ++-- www/drive/inner.js | 6 ++++-- www/poll/inner.js | 6 ++++-- www/teams/inner.js | 4 ++-- 7 files changed, 34 insertions(+), 17 deletions(-) diff --git a/www/common/common-interface.js b/www/common/common-interface.js index 04f806d4f..f08a9aa72 100644 --- a/www/common/common-interface.js +++ b/www/common/common-interface.js @@ -491,6 +491,11 @@ define([ $ok.focus(); Notifier.notify(); }); + + return { + element: frame, + delete: close + }; }; UI.prompt = function (msg, def, cb, opt, force) { diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index c60ef652d..d106702b5 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -56,6 +56,21 @@ define([ }); }; + var dcAlert; + UIElements.disconnectAlert = function () { + if (dcAlert && $(dcAlert.element).length) { return; } + dcAlert = UI.alert(Messages.common_connectionLost, undefined, true); + }; + UIElements.reconnectAlert = function () { + if (!dcAlert) { return; } + if (!dcAlert.delete) { + dcAlert = undefined; + return; + } + dcAlert.delete(); + dcAlert = undefined; + }; + var importContent = function (type, f, cfg) { return function () { var $files = $('', {type:"file"}); diff --git a/www/common/onlyoffice/inner.js b/www/common/onlyoffice/inner.js index 8b2d8328b..4270b0f4c 100644 --- a/www/common/onlyoffice/inner.js +++ b/www/common/onlyoffice/inner.js @@ -1606,17 +1606,10 @@ define([ pinImages(); }; - config.onAbort = function () { - // inform of network disconnect - setEditable(false); - toolbar.failed(); - UI.alert(Messages.common_connectionLost, undefined, true); - }; - config.onConnectionChange = function (info) { if (info.state) { // If we tried to send changes while we were offline, force a page reload - UI.findOKButton().click(); + UIElements.reconnectAlert(); if (Object.keys(pendingChanges).length) { return void UI.confirm(Messages.oo_reconnect, function (yes) { if (!yes) { return; } @@ -1629,7 +1622,7 @@ define([ setEditable(false); offline = true; UI.findOKButton().click(); - UI.alert(Messages.common_connectionLost, undefined, true); + UIElements.disconnectAlert(); } }; diff --git a/www/common/sframe-app-framework.js b/www/common/sframe-app-framework.js index ed8652809..ea22aaf5f 100644 --- a/www/common/sframe-app-framework.js +++ b/www/common/sframe-app-framework.js @@ -396,9 +396,9 @@ define([ if (state === STATE.DELETED) { return; } stateChange(info.state ? STATE.INITIALIZING : STATE.DISCONNECTED, info.permanent); /*if (info.state) { - UI.findOKButton().click(); + UIElements.reconnectAlert(); } else { - UI.alert(Messages.common_connectionLost, undefined, true); + UIElements.disconnectAlert(); }*/ }; diff --git a/www/drive/inner.js b/www/drive/inner.js index 03bf73697..a5325a651 100644 --- a/www/drive/inner.js +++ b/www/drive/inner.js @@ -5,6 +5,7 @@ define([ '/common/common-util.js', '/common/common-hash.js', '/common/common-interface.js', + '/common/common-ui-elements.js', '/common/common-feedback.js', '/bower_components/nthen/index.js', '/common/sframe-common.js', @@ -22,6 +23,7 @@ define([ Util, Hash, UI, + UIElements, Feedback, nThen, SFCommon, @@ -272,13 +274,13 @@ define([ setEditable(false); if (drive.refresh) { drive.refresh(); } APP.toolbar.failed(); - if (!noAlert) { UI.alert(Messages.common_connectionLost, undefined, true); } + if (!noAlert) { UIElements.disconnectAlert(); } }; var onReconnect = function () { setEditable(true); if (drive.refresh) { drive.refresh(); } APP.toolbar.reconnecting(); - UI.findOKButton().click(); + UIElements.reconnectAlert(); }; sframeChan.on('EV_DRIVE_LOG', function (msg) { diff --git a/www/poll/inner.js b/www/poll/inner.js index 7420f5c09..d1d583757 100644 --- a/www/poll/inner.js +++ b/www/poll/inner.js @@ -13,6 +13,7 @@ define([ '/common/sframe-common-codemirror.js', '/common/common-thumbnail.js', '/common/common-interface.js', + '/common/common-ui-elements.js', '/common/hyperscript.js', '/customize/messages.js', 'cm/lib/codemirror', @@ -42,6 +43,7 @@ define([ SframeCM, Thumb, UI, + UIElements, h, Messages, CMeditor, @@ -1098,13 +1100,13 @@ define([ }); } setEditable(false); - //UI.alert(Messages.common_connectionLost, undefined, true); + //UIElements.disconnectAlert(); }; var onReconnect = function () { if (APP.unrecoverable) { return; } setEditable(true); - //UI.findOKButton().click(); + //UIElements.reconnectAlert(); }; var getHeadingText = function () { diff --git a/www/teams/inner.js b/www/teams/inner.js index 3f26874b2..4dea3e80d 100644 --- a/www/teams/inner.js +++ b/www/teams/inner.js @@ -1379,13 +1379,13 @@ define([ setEditable(false); if (APP.team && driveAPP.refresh) { driveAPP.refresh(); } toolbar.failed(); - if (!noAlert) { UI.alert(Messages.common_connectionLost, undefined, true); } + if (!noAlert) { UIElements.disconnectAlert(); } }; var onReconnect = function () { setEditable(true); if (APP.team && driveAPP.refresh) { driveAPP.refresh(); } toolbar.reconnecting(); - UI.findOKButton().click(); + UIElements.reconnectAlert(); }; sframeChan.on('EV_DRIVE_LOG', function (msg) {