From 10eed5c46dc6dc63a9348be73e8f3ce112074cb8 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 13 Jan 2020 10:28:28 -0500 Subject: [PATCH 001/172] drop unmaintained flow annotations --- storage/file.js | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/storage/file.js b/storage/file.js index bb65cff43..74f794cca 100644 --- a/storage/file.js +++ b/storage/file.js @@ -867,36 +867,7 @@ var getMessages = function (env, chanName, handler, cb) { }); }; -/*:: -export type ChainPadServer_MessageObj_t = { buff: Buffer, offset: number }; -export type ChainPadServer_Storage_t = { - readMessagesBin: ( - channelName:string, - start:number, - asyncMsgHandler:(msg:ChainPadServer_MessageObj_t, moreCb:()=>void, abortCb:()=>void)=>void, - cb:(err:?Error)=>void - )=>void, - message: (channelName:string, content:string, cb:(err:?Error)=>void)=>void, - messageBin: (channelName:string, content:Buffer, cb:(err:?Error)=>void)=>void, - getMessages: (channelName:string, msgHandler:(msg:string)=>void, cb:(err:?Error)=>void)=>void, - removeChannel: (channelName:string, cb:(err:?Error)=>void)=>void, - closeChannel: (channelName:string, cb:(err:?Error)=>void)=>void, - flushUnusedChannels: (cb:()=>void)=>void, - getChannelSize: (channelName:string, cb:(err:?Error, size:?number)=>void)=>void, - getChannelMetadata: (channelName:string, cb:(err:?Error|string, data:?any)=>void)=>void, - clearChannel: (channelName:string, (err:?Error)=>void)=>void -}; -export type ChainPadServer_Config_t = { - verbose?: boolean, - filePath?: string, - channelExpirationMs?: number, - openFileLimit?: number -}; -*/ -module.exports.create = function ( - conf /*:ChainPadServer_Config_t*/, - cb /*:(store:ChainPadServer_Storage_t)=>void*/ -) { +module.exports.create = function (conf, cb) { var env = { root: conf.filePath || './datastore', archiveRoot: conf.archivePath || './data/archive', From c89595c7bbab8e37c31e3ba420a64d948b0b9664 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 13 Jan 2020 10:32:06 -0500 Subject: [PATCH 002/172] handle all the simple cases where operations on channels should be queued --- storage/file.js | 68 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/storage/file.js b/storage/file.js index 74f794cca..2a8802c8c 100644 --- a/storage/file.js +++ b/storage/file.js @@ -7,6 +7,8 @@ var Path = require("path"); var nThen = require("nthen"); var Semaphore = require("saferphore"); var Util = require("../lib/common-util"); + +const WriteQueue = require("../lib/write-queue"); const Readline = require("readline"); const ToPull = require('stream-to-pull-stream'); const Pull = require('pull-stream'); @@ -169,6 +171,7 @@ var readMessages = function (path, msgHandler, cb) { could have been amended */ var getChannelMetadata = function (Env, channelId, cb) { + // XXX queue var path = mkPath(Env, channelId); // gets metadata embedded in a file @@ -177,6 +180,7 @@ var getChannelMetadata = function (Env, channelId, cb) { // low level method for getting just the dedicated metadata channel var getDedicatedMetadata = function (env, channelId, handler, cb) { + // XXX queue var metadataPath = mkMetadataPath(env, channelId); readMessages(metadataPath, function (line) { if (!line) { return; } @@ -219,6 +223,7 @@ How to proceed */ + // XXX queue nThen(function (w) { // returns the first line of a channel, parsed... getChannelMetadata(env, channelId, w(function (err, data) { @@ -263,6 +268,7 @@ var writeMetadata = function (env, channelId, data, cb) { // TODO see if we could improve performance by using libnewline const NEWLINE_CHR = ('\n').charCodeAt(0); const mkBufferSplit = () => { + // XXX queue lock let remainder = null; return Pull((read) => { return (abort, cb) => { @@ -314,6 +320,7 @@ const mkOffsetCounter = () => { const readMessagesBin = (env, id, start, msgHandler, cb) => { const stream = Fs.createReadStream(mkPath(env, id), { start: start }); let keepReading = true; + // XXX queue lock Pull( ToPull.read(stream), mkBufferSplit(), @@ -837,6 +844,7 @@ var message = function (env, chanName, msg, cb) { // stream messages from a channel log var getMessages = function (env, chanName, handler, cb) { + // XXX queue lock getChannel(env, chanName, function (err, chan) { if (!chan) { cb(err); @@ -878,6 +886,7 @@ module.exports.create = function (conf, cb) { openFiles: 0, openFileLimit: conf.openFileLimit || 2048, }; + var queue = env.queue = WriteQueue(); var it; nThen(function (w) { @@ -899,10 +908,13 @@ module.exports.create = function (conf, cb) { // write a new message to a log message: function (channelName, content, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - message(env, channelName, content, cb); + queue(channelName, function (next) { + message(env, channelName, content, Util.both(cb, next)); + }); }, // iterate over all the messages in a log getMessages: function (channelName, msgHandler, cb) { + // XXX queue lock if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } getMessages(env, channelName, msgHandler, cb); }, @@ -911,10 +923,13 @@ module.exports.create = function (conf, cb) { // write a new message to a log messageBin: (channelName, content, cb) => { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - messageBin(env, channelName, content, cb); + queue(channelName, function (next) { + messageBin(env, channelName, content, Util.both(cb, next)); + }); }, // iterate over the messages in a log readMessagesBin: (channelName, start, asyncMsgHandler, cb) => { + // XXX queue lock if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } readMessagesBin(env, channelName, start, asyncMsgHandler, cb); }, @@ -923,19 +938,23 @@ module.exports.create = function (conf, cb) { // remove a channel and its associated metadata log if present removeChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - removeChannel(env, channelName, function (err) { - cb(err); + queue(channelName, function (next) { + removeChannel(env, channelName, Util.both(cb, next)); }); }, // remove a channel and its associated metadata log from the archive directory removeArchivedChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - removeArchivedChannel(env, channelName, cb); + queue(channelName, function (next) { + removeArchivedChannel(env, channelName, Util.both(cb, next)); + }); }, // clear all data for a channel but preserve its metadata clearChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - clearChannel(env, channelName, cb); + queue(channelName, function (next) { + clearChannel(env, channelName, Util.both(cb, next)); + }); }, // check if a channel exists in the database @@ -943,47 +962,62 @@ module.exports.create = function (conf, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } // construct the path var filepath = mkPath(env, channelName); - channelExists(filepath, cb); + queue(channelName, function (next) { + channelExists(filepath, Util.both(cb, next)); + }); }, // check if a channel exists in the archive isChannelArchived: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } // construct the path var filepath = mkArchivePath(env, channelName); - channelExists(filepath, cb); + queue(channelName, function (next) { + channelExists(filepath, Util.both(cb, next)); + }); }, // move a channel from the database to the archive, along with its metadata archiveChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - archiveChannel(env, channelName, cb); + queue(channelName, function (next) { + archiveChannel(env, channelName, Util.both(cb, next)); + }); }, // restore a channel from the archive to the database, along with its metadata restoreArchivedChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - unarchiveChannel(env, channelName, cb); + queue(channelName, function (next) { + unarchiveChannel(env, channelName, Util.both(cb, next)); + }); }, // METADATA METHODS // fetch the metadata for a channel getChannelMetadata: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - getChannelMetadata(env, channelName, cb); + queue(channelName, function (next) { + getChannelMetadata(env, channelName, Util.both(cb, next)); + }); }, // iterate over lines of metadata changes from a dedicated log readDedicatedMetadata: function (channelName, handler, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - getDedicatedMetadata(env, channelName, handler, cb); + queue(channelName, function (next) { + getDedicatedMetadata(env, channelName, handler, Util.both(cb, next)); + }); }, // iterate over multiple lines of metadata changes readChannelMetadata: function (channelName, handler, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } + // XXX queue readMetadata(env, channelName, handler, cb); }, // write a new line to a metadata log writeMetadata: function (channelName, data, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - writeMetadata(env, channelName, data, cb); + queue(channelName, function (next) { + writeMetadata(env, channelName, data, Util.both(cb, next)); + }); }, // CHANNEL ITERATION @@ -996,13 +1030,17 @@ module.exports.create = function (conf, cb) { getChannelSize: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - channelBytes(env, channelName, cb); + queue(channelName, function (next) { + channelBytes(env, channelName, Util.both(cb, next)); + }); }, // OTHER DATABASE FUNCTIONALITY // remove a particular channel from the cache closeChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - closeChannel(env, channelName, cb); + queue(channelName, function (next) { + closeChannel(env, channelName, Util.both(cb, next)); + }); }, // iterate over open channels and close any that are not active flushUnusedChannels: function (cb) { From 47fdf9de9f8d484b93a710927f1d65688af782b5 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 13 Jan 2020 10:32:31 -0500 Subject: [PATCH 003/172] add a note to fix some odd behaviour --- storage/file.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/file.js b/storage/file.js index 2a8802c8c..eca3d76be 100644 --- a/storage/file.js +++ b/storage/file.js @@ -759,6 +759,8 @@ var getChannel = function ( } if (env.openFiles >= env.openFileLimit) { + // FIXME warn if this is the case? + // alternatively use graceful-fs to handle lots of concurrent reads // if you're running out of open files, asynchronously clean up expired files // do it on a shorter timeframe, though (half of normal) setTimeout(function () { From 7072fe4fa4b6eebd7b693bcbce657bb22f3398d6 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 15 Jan 2020 13:32:52 -0500 Subject: [PATCH 004/172] implement and test a complex constrained scheduler --- lib/schedule.js | 173 +++++++++++++++++++++++++ scripts/tests/test-scheduler.js | 220 ++++++++++++++++++++++++++++++++ 2 files changed, 393 insertions(+) create mode 100644 lib/schedule.js create mode 100644 scripts/tests/test-scheduler.js diff --git a/lib/schedule.js b/lib/schedule.js new file mode 100644 index 000000000..1e93a2126 --- /dev/null +++ b/lib/schedule.js @@ -0,0 +1,173 @@ +var WriteQueue = require("./write-queue"); +var Util = require("./common-util"); + +/* This module provides implements a FIFO scheduler + which assumes the existence of three types of async tasks: + + 1. ordered tasks which must be executed sequentially + 2. unordered tasks which can be executed in parallel + 3. blocking tasks which must block the execution of all other tasks + + The scheduler assumes there will be many resources identified by strings, + and that the constraints described above will only apply in the context + of identical string ids. + + Many blocking tasks may be executed in parallel so long as they + concern resources identified by different ids. + +USAGE: + + const schedule = require("./schedule")(); + + // schedule two sequential tasks using the resource 'pewpew' + schedule.ordered('pewpew', function (next) { + appendToFile('beep\n', next); + }); + schedule.ordered('pewpew', function (next) { + appendToFile('boop\n', next); + }); + + // schedule a task that can happen whenever + schedule.unordered('pewpew', function (next) { + displayFileSize(next); + }); + + // schedule a blocking task which will wait + // until the all unordered tasks have completed before commencing + schedule.blocking('pewpew', function (next) { + deleteFile(next); + }); + + // this will be queued for after the blocking task + schedule.ordered('pewpew', function (next) { + appendFile('boom', next); + }); + +*/ + +// return a uid which is not already in a map +var unusedUid = function (set) { + var uid = Util.uid(); + if (set[uid]) { return unusedUid(); } + return uid; +}; + +// return an existing session, creating one if it does not already exist +var lookup = function (map, id) { + return (map[id] = map[id] || { + //blocking: [], + active: {}, + blocked: {}, + }); +}; + +var isEmpty = function (map) { + for (var key in map) { + if (map.hasOwnProperty(key)) { return false; } + } + return true; +}; + +// XXX enforce asynchrony everywhere +module.exports = function () { + // every scheduler instance has its own queue + var queue = WriteQueue(); + + // ordered tasks don't require any extra logic + var Ordered = function (id, task) { + queue(id, task); + }; + + // unordered and blocking tasks need a little extra state + var map = {}; + + // regular garbage collection keeps memory consumption low + var collectGarbage = function (id) { + // avoid using 'lookup' since it creates a session implicitly + var local = map[id]; + // bail out if no session + if (!local) { return; } + // bail out if there are blocking or active tasks + if (local.lock) { return; } + if (!isEmpty(local.active)) { return; } + // if there are no pending actions then delete the session + delete map[id]; + }; + + // unordered tasks run immediately if there are no blocking tasks scheduled + // or immediately after blocking tasks finish + var runImmediately = function (local, task) { + // set a flag in the map of active unordered tasks + // to prevent blocking tasks from running until you finish + var uid = unusedUid(local.active); + local.active[uid] = true; + + task(function () { + // remove the flag you set to indicate that your task completed + delete local.active[uid]; + // don't do anything if other unordered tasks are still running + if (!isEmpty(local.active)) { return; } + // bail out if there are no blocking tasks scheduled or ready + if (typeof(local.waiting) !== 'function') { + return void collectGarbage(); + } + local.waiting(); + }); + }; + + var runOnceUnblocked = function (local, task) { + var uid = unusedUid(local.blocked); + local.blocked[uid] = function () { + runImmediately(local, task); + }; + }; + + // 'unordered' tasks are scheduled to run in after the most recently received blocking task + // or immediately and in parallel if there are no blocking tasks scheduled. + var Unordered = function (id, task) { + var local = lookup(map, id); + if (local.lock) { return runOnceUnblocked(local, task); } + runImmediately(local, task); + }; + + var runBlocked = function (local) { + for (var task in local.blocked) { + runImmediately(local, local.blocked[task]); + } + }; + + // 'blocking' tasks must be run alone. + // They are queued alongside ordered tasks, + // and wait until any running 'unordered' tasks complete before commencing. + var Blocking = function (id, task) { + var local = lookup(map, id); + + queue(id, function (next) { + // start right away if there are no running unordered tasks + if (isEmpty(local.active)) { + local.lock = true; + return void task(function () { + delete local.lock; + runBlocked(local); + next(); + }); + } + // otherwise wait until the running tasks have completed + local.waiting = function () { + local.lock = true; + task(function () { + delete local.lock; + delete local.waiting; + runBlocked(local); + next(); + }); + }; + }); + }; + + return { + ordered: Ordered, + unordered: Unordered, + blocking: Blocking, + }; +}; diff --git a/scripts/tests/test-scheduler.js b/scripts/tests/test-scheduler.js new file mode 100644 index 000000000..6a076d5aa --- /dev/null +++ b/scripts/tests/test-scheduler.js @@ -0,0 +1,220 @@ +/* three types of actions: + * read + * write + * append + each of which take a random amount of time + +*/ +var Util = require("../../lib/common-util"); +var schedule = require("../../lib/schedule")(); +var nThen = require("nthen"); + +var rand = function (n) { + return Math.floor(Math.random() * n); +}; + +var rand_time = function () { + // between 51 and 151 + return rand(300) + 25; +}; + +var makeAction = function (type) { + var i = 0; + return function (time) { + var j = i++; + return function (next) { + console.log(" Beginning action: %s#%s", type, j); + setTimeout(function () { + console.log(" Completed action: %s#%s", type, j); + next(); + }, time); + return j; + }; + }; +}; + +var TYPES = ['WRITE', 'READ', 'APPEND']; +var chooseAction = function () { + var n = rand(100); + + if (n < 50) { return 'APPEND'; } + if (n < 90) { return 'READ'; } + return 'WRITE'; + + //return TYPES[rand(3)]; +}; + +var test = function (script, cb) { + var uid = Util.uid(); + + var TO_RUN = script.length; + var total_run = 0; + + var parallel = 0; + var last_run_ordered = -1; + //var i = 0; + + var ACTIONS = {}; + TYPES.forEach(function (type) { + ACTIONS[type] = makeAction(type); + }); + + nThen(function (w) { + setTimeout(w(), 3000); + // run scripted actions with assertions + script.forEach(function (scene) { + var type = scene[0]; + var time = typeof(scene[1]) === 'number'? scene[1]: rand_time(); + + var action = ACTIONS[type](time); + console.log("Queuing action of type: %s(%s)", type, time); + + var proceed = w(); + + switch (type) { + case 'APPEND': + return schedule.ordered(uid, w(function (next) { + parallel++; + var temp = action(function () { + parallel--; + total_run++; + proceed(); + next(); + }); + if (temp !== (last_run_ordered + 1)) { + throw new Error("out of order"); + } + last_run_ordered = temp; + })); + case 'WRITE': + return schedule.blocking(uid, w(function (next) { + parallel++; + action(function () { + parallel--; + total_run++; + proceed(); + next(); + }); + if (parallel > 1) { + console.log("parallelism === %s", parallel); + throw new Error("too much parallel"); + } + })); + case 'READ': + return schedule.unordered(uid, w(function (next) { + parallel++; + action(function () { + parallel--; + total_run++; + proceed(); + next(); + }); + })); + default: + throw new Error("wut"); + } + }); + }).nThen(function () { + // make assertions about the whole script + if (total_run !== TO_RUN) { + console.log("Ran %s / %s", total_run, TO_RUN); + throw new Error("skipped tasks"); + } + console.log("total_run === %s", total_run); + + cb(); + }); +}; + + +var randomScript = function () { + var len = rand(15) + 10; + var script = []; + while (len--) { + script.push([ + chooseAction(), + rand_time(), + ]); + } + return script; +}; + +var WRITE = function (t) { + return ['WRITE', t]; +}; +var READ = function (t) { + return ['READ', t]; +}; + +var APPEND = function (t) { + return ['APPEND', t]; +}; + +nThen(function (w) { + test([ + ['READ', 150], + ['APPEND', 200], + ['APPEND', 100], + ['READ', 350], + ['WRITE', 400], + ['APPEND', 275], + ['APPEND', 187], + ['WRITE', 330], + ['WRITE', 264], + ['WRITE', 256], + ], w(function () { + console.log("finished pre-scripted test\n"); + })); +}).nThen(function (w) { + test([ + WRITE(289), + APPEND(281), + READ(207), + WRITE(225), + READ(279), + WRITE(300), + READ(331), + APPEND(341), + APPEND(385), + READ(313), + WRITE(285), + READ(304), + APPEND(273), + APPEND(150), + WRITE(246), + READ(244), + WRITE(172), + APPEND(253), + READ(215), + READ(296), + APPEND(281), + APPEND(296), + WRITE(168), + ], w(function () { + console.log("finished 2nd pre-scripted test\n"); + })); +}).nThen(function () { + var totalTests = 50; + var randomTests = 1; + + var last = nThen(function () { + console.log("beginning randomized tests"); + }); + + var queueRandomTest = function (i) { + last = last.nThen(function (w) { + console.log("running random test script #%s\n", i); + test(randomScript(), w(function () { + console.log("finished random test #%s\n", i); + })); + }); + }; + + while (randomTests <=totalTests) { queueRandomTest(randomTests++); } + + last.nThen(function () { + console.log("finished %s random tests", totalTests); + }); +}); + + From 15ca855f221d07653e8b01ff7af0fa0488600471 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 15 Jan 2020 15:09:30 -0500 Subject: [PATCH 005/172] start using the scheduler for all relevant database methods ...and describe the rationale for using particular scheduler semantics --- storage/file.js | 143 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 114 insertions(+), 29 deletions(-) diff --git a/storage/file.js b/storage/file.js index eca3d76be..b2899619f 100644 --- a/storage/file.js +++ b/storage/file.js @@ -8,7 +8,7 @@ var nThen = require("nthen"); var Semaphore = require("saferphore"); var Util = require("../lib/common-util"); -const WriteQueue = require("../lib/write-queue"); +const Schedule = require("../lib/schedule"); const Readline = require("readline"); const ToPull = require('stream-to-pull-stream'); const Pull = require('pull-stream'); @@ -171,7 +171,6 @@ var readMessages = function (path, msgHandler, cb) { could have been amended */ var getChannelMetadata = function (Env, channelId, cb) { - // XXX queue var path = mkPath(Env, channelId); // gets metadata embedded in a file @@ -180,7 +179,6 @@ var getChannelMetadata = function (Env, channelId, cb) { // low level method for getting just the dedicated metadata channel var getDedicatedMetadata = function (env, channelId, handler, cb) { - // XXX queue var metadataPath = mkMetadataPath(env, channelId); readMessages(metadataPath, function (line) { if (!line) { return; } @@ -223,7 +221,6 @@ How to proceed */ - // XXX queue nThen(function (w) { // returns the first line of a channel, parsed... getChannelMetadata(env, channelId, w(function (err, data) { @@ -268,7 +265,6 @@ var writeMetadata = function (env, channelId, data, cb) { // TODO see if we could improve performance by using libnewline const NEWLINE_CHR = ('\n').charCodeAt(0); const mkBufferSplit = () => { - // XXX queue lock let remainder = null; return Pull((read) => { return (abort, cb) => { @@ -320,7 +316,6 @@ const mkOffsetCounter = () => { const readMessagesBin = (env, id, start, msgHandler, cb) => { const stream = Fs.createReadStream(mkPath(env, id), { start: start }); let keepReading = true; - // XXX queue lock Pull( ToPull.read(stream), mkBufferSplit(), @@ -846,7 +841,6 @@ var message = function (env, chanName, msg, cb) { // stream messages from a channel log var getMessages = function (env, chanName, handler, cb) { - // XXX queue lock getChannel(env, chanName, function (err, chan) { if (!chan) { cb(err); @@ -888,9 +882,26 @@ module.exports.create = function (conf, cb) { openFiles: 0, openFileLimit: conf.openFileLimit || 2048, }; - var queue = env.queue = WriteQueue(); var it; + /* our scheduler prioritizes and executes tasks with respect + to all other tasks invoked with an identical key + (typically the id of the concerned channel) + + it assumes that all tasks can be categorized into three types + + 1. unordered tasks such as streaming reads which can take + a long time to complete. + + 2. ordered tasks such as appending to a file which does not + take very long, but where priority is important. + + 3. blocking tasks such as rewriting a file where it would be + dangerous to perform any other task concurrently. + + */ + var schedule = env.schedule = Schedule(); + nThen(function (w) { // make sure the store's directory exists Fse.mkdirp(env.root, PERMISSIVE, w(function (err) { @@ -910,61 +921,112 @@ module.exports.create = function (conf, cb) { // write a new message to a log message: function (channelName, content, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { + schedule.ordered(channelName, function (next) { message(env, channelName, content, Util.both(cb, next)); }); }, // iterate over all the messages in a log getMessages: function (channelName, msgHandler, cb) { - // XXX queue lock if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - getMessages(env, channelName, msgHandler, cb); + schedule.unordered(channelName, function (next) { + getMessages(env, channelName, msgHandler, Util.both(cb, next)); + }); }, // NEWER IMPLEMENTATIONS OF THE SAME THING // write a new message to a log messageBin: (channelName, content, cb) => { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { + schedule.ordered(channelName, function (next) { messageBin(env, channelName, content, Util.both(cb, next)); }); }, // iterate over the messages in a log readMessagesBin: (channelName, start, asyncMsgHandler, cb) => { - // XXX queue lock if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - readMessagesBin(env, channelName, start, asyncMsgHandler, cb); +// XXX there is a race condition here +// historyKeeper reads the file to find the byte offset of the first interesting message +// then calls this function again to read from that point. +// If this task is in the queue already when the file is read again +// then that byte offset will have been invalidated +// and the resulting stream probably won't align with message boundaries. +// We can evict the cache in the callback but by that point it will be too late. +// Presumably we'll need to bury some of historyKeeper's logic into a filestore method +// in order to make index/read sequences atomic. +// Otherwise, we can add a new task type to the scheduler to take invalidation into account... +// either method introduces significant complexity. + schedule.unordered(channelName, function (next) { + readMessagesBin(env, channelName, start, asyncMsgHandler, Util.both(cb, next)); + }); }, // METHODS for deleting data // remove a channel and its associated metadata log if present removeChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// XXX there's another race condition here... +// when a remove and an append are scheduled in that order +// the remove will delete the channel's metadata (including its validateKey) +// then the append will recreate the channel and insert a message. +// clients that are connected to the channel via historyKeeper should be kicked out +// however, anyone that connects to that channel in the future will be able to read the +// signed message, but will not find its validate key... +// resulting in a junk/unusable document + schedule.ordered(channelName, function (next) { removeChannel(env, channelName, Util.both(cb, next)); }); }, // remove a channel and its associated metadata log from the archive directory removeArchivedChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { + schedule.ordered(channelName, function (next) { removeArchivedChannel(env, channelName, Util.both(cb, next)); }); }, // clear all data for a channel but preserve its metadata clearChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { + schedule.ordered(channelName, function (next) { clearChannel(env, channelName, Util.both(cb, next)); }); }, + trimChannel: function (channelName, hash, cb) { + // XXX ansuz + // XXX queue lock + /* block any reads from the metadata and log files + until this whole process has finished + close the file descriptor if it is open + derive temporary file paths for metadata and log buffers + compute metadata state and write to metadata buffer + scan through log file and begin copying lines to the log buffer + once you recognize the first line by the hash the user provided + archive the file and current metadata once both buffers are copied + move the metadata and log buffers into place + return the lock on reads + call back + + in case of an error, remove the buffer files + */ + schedule.blocking(channelName, function (next) { + cb("E_NOT_IMPLEMENTED"); + next(); + }); + }, // check if a channel exists in the database isChannelAvailable: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } // construct the path var filepath = mkPath(env, channelName); - queue(channelName, function (next) { +// (ansuz) I'm uncertain whether this task should be unordered or ordered. +// there's a round trip to the client (and possibly the user) before they decide +// to act on the information of whether there is already content present in this channel. +// so it's practically impossible to avoid race conditions where someone else creates +// some content before you. +// if that's the case, it's basically impossible that you'd generate the same signing key, +// and thus historykeeper should reject the signed messages of whoever loses the race. +// thus 'unordered' seems appropriate. + schedule.unordered(channelName, function (next) { channelExists(filepath, Util.both(cb, next)); }); }, @@ -973,21 +1035,30 @@ module.exports.create = function (conf, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } // construct the path var filepath = mkArchivePath(env, channelName); - queue(channelName, function (next) { +// as with the method above, somebody might remove, restore, or overwrite an archive +// in the time that it takes to answer this query and to execute whatever follows. +// since it's impossible to win the race every time let's just make this 'unordered' + schedule.unordered(channelName, function (next) { channelExists(filepath, Util.both(cb, next)); }); }, // move a channel from the database to the archive, along with its metadata archiveChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// again, the semantics around archiving and appending are really muddy. +// so I'm calling this 'unordered' again + schedule.unordered(channelName, function (next) { archiveChannel(env, channelName, Util.both(cb, next)); }); }, // restore a channel from the archive to the database, along with its metadata restoreArchivedChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// archive restoration will fail if either a file or its metadata exists in the live db. +// so I'm calling this 'ordered' to give writes a chance to flush out. +// accidental conflicts are extremely unlikely since clients check the status +// of a previously known channel before joining. + schedule.ordered(channelName, function (next) { unarchiveChannel(env, channelName, Util.both(cb, next)); }); }, @@ -996,14 +1067,17 @@ module.exports.create = function (conf, cb) { // fetch the metadata for a channel getChannelMetadata: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// The only thing that can invalid this method's results are channel archival, removal, or trimming. +// We want it to be fast, so let's make it unordered. + schedule.unordered(channelName, function (next) { getChannelMetadata(env, channelName, Util.both(cb, next)); }); }, // iterate over lines of metadata changes from a dedicated log readDedicatedMetadata: function (channelName, handler, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// Everything that modifies metadata also updates clients, so this can be 'unordered' + schedule.unordered(channelName, function (next) { getDedicatedMetadata(env, channelName, handler, Util.both(cb, next)); }); }, @@ -1011,13 +1085,16 @@ module.exports.create = function (conf, cb) { // iterate over multiple lines of metadata changes readChannelMetadata: function (channelName, handler, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - // XXX queue - readMetadata(env, channelName, handler, cb); +// same logic as 'readDedicatedMetadata + schedule.unordered(channelName, function (next) { + readMetadata(env, channelName, handler, Util.both(cb, next)); + }); }, // write a new line to a metadata log writeMetadata: function (channelName, data, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// metadata writes are fast and should be applied in order + schedule.ordered(channelName, function (next) { writeMetadata(env, channelName, data, Util.both(cb, next)); }); }, @@ -1032,7 +1109,9 @@ module.exports.create = function (conf, cb) { getChannelSize: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// this method should be really fast and it probably doesn't matter much +// if we get the size slightly before or after somebody writes a few hundred bytes to it. + schedule.ordered(channelName, function (next) { channelBytes(env, channelName, Util.both(cb, next)); }); }, @@ -1040,7 +1119,10 @@ module.exports.create = function (conf, cb) { // remove a particular channel from the cache closeChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - queue(channelName, function (next) { +// It is most likely the case that the channel is inactive if we are trying to close it, +// thus it doesn't make much difference whether it's ordered or not. +// In any case, it will be re-opened if anyone tries to write to it. + schedule.ordered(channelName, function (next) { closeChannel(env, channelName, Util.both(cb, next)); }); }, @@ -1050,7 +1132,10 @@ module.exports.create = function (conf, cb) { }, // write to a log file log: function (channelName, content, cb) { - message(env, channelName, content, cb); +// you probably want the events in your log to be in the correct order. + schedule.ordered(channelName, function (next) { + message(env, channelName, content, Util.both(cb, next)); + }); }, // shut down the database shutdown: function () { From ff40538ee7b2d8e033533d62be49341cdfb92425 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 15 Jan 2020 15:11:45 -0500 Subject: [PATCH 006/172] sketch our two new RPCs for trimming history --- rpc.js | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/rpc.js b/rpc.js index fcd85a390..8041cdcad 100644 --- a/rpc.js +++ b/rpc.js @@ -933,6 +933,16 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { }); }; +var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { + // XXX validate that the user sending the request owns the channel in question + // proceed to call Env.msgStore.trimChannel(channelId, hash, cb) if ok + // otherwise reject with INSUFFICIENT_PERMISSIONS + + // XXX if trimChannel calls back without an error + // you must also clear the channel's index from historyKeeper cache + cb("E_NOT_IMPLEMENTED"); +}; + /* Users should be able to clear their own pin log with an authenticated RPC */ var removePins = function (Env, safeKey, cb) { @@ -949,6 +959,11 @@ var removePins = function (Env, safeKey, cb) { }); }; +var trimPins = function (Env, safeKey, cb) { + // XXX trim to latest pin checkpoint + cb("NOT_IMPLEMENTED"); +}; + /* We assume that the server is secured against MitM attacks via HTTPS, and that malicious actors do not have code execution @@ -1313,9 +1328,11 @@ var isAuthenticatedCall = function (call) { 'OWNED_UPLOAD_COMPLETE', 'UPLOAD_CANCEL', 'EXPIRE_SESSION', + 'TRIM_OWNED_CHANNEL_HISTORY', 'CLEAR_OWNED_CHANNEL', 'REMOVE_OWNED_CHANNEL', 'REMOVE_PINS', + 'TRIM_PINS', 'WRITE_LOGIN_BLOCK', 'REMOVE_LOGIN_BLOCK', 'ADMIN', @@ -1635,11 +1652,21 @@ RPC.create = function ( if (e) { return void Respond(e); } Respond(void 0, "OK"); }); + case 'TRIM_OWNED_CHANNEL_HISTORY': + return void removeOwnedChannelHistory(Env, msg[1], publicKey, msg[2], function (e) { + if (e) { return void Respond(e); } + Respond(void 0, 'OK'); + }); case 'REMOVE_PINS': return void removePins(Env, safeKey, function (e) { if (e) { return void Respond(e); } Respond(void 0, "OK"); }); + case 'TRIM_PINS': + return void trimPins(Env, safeKey, function (e) { + if (e) { return void Respond(e); } + Respond(void 0, "OK"); + }); case 'UPLOAD': return void Env.blobStore.upload(safeKey, msg[1], function (e, len) { WARN(e, len); From 4418f6a113245cb009620c57902c9e4078900aa2 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 16 Jan 2020 14:24:07 -0500 Subject: [PATCH 007/172] tiny optimization which saves a little bit of memory usage for a little bit of time --- historyKeeper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/historyKeeper.js b/historyKeeper.js index 59a694e8c..5f9c48c93 100644 --- a/historyKeeper.js +++ b/historyKeeper.js @@ -284,9 +284,9 @@ module.exports.create = function (cfg) { const storeMessage = function (ctx, channel, msg, isCp, optionalMessageHash) { const id = channel.id; - const msgBin = Buffer.from(msg + '\n', 'utf8'); queueStorage(id, function (next) { + const msgBin = Buffer.from(msg + '\n', 'utf8'); // Store the message first, and update the index only once it's stored. // store.messageBin can be async so updating the index first may // result in a wrong cpIndex From 8c5c643a25ff2b3e7d90e234d254af329926ccec Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 16 Jan 2020 14:25:00 -0500 Subject: [PATCH 008/172] finally get around to reorganizing the messiest part of history keeper --- historyKeeper.js | 416 ++++++++++++++++++++++++----------------------- 1 file changed, 213 insertions(+), 203 deletions(-) diff --git a/historyKeeper.js b/historyKeeper.js index 5f9c48c93..fe16a204c 100644 --- a/historyKeeper.js +++ b/historyKeeper.js @@ -730,227 +730,204 @@ module.exports.create = function (cfg) { } }; - /* onDirectMessage - * exported for use by the netflux-server - * parses and handles all direct messages directed to the history keeper - * check if it's expired and execute all the associated side-effects - * routes queries to the appropriate handlers - * GET_HISTORY - * GET_HISTORY_RANGE - * GET_FULL_HISTORY - * RPC - * if the rpc has special hooks that the history keeper needs to be aware of... - * execute them here... - - */ - const onDirectMessage = function (ctx, seq, user, json) { - let parsed; - let channelName; - - Log.silly('HK_MESSAGE', json); - - try { - parsed = JSON.parse(json[2]); - } catch (err) { - Log.error("HK_PARSE_CLIENT_MESSAGE", json); - return; - } - - // If the requested history is for an expired channel, abort - // Note the if we don't have the keys for that channel in metadata_cache, we'll - // have to abort later (once we know the expiration time) - if (checkExpired(ctx, parsed[1])) { return; } - - if (parsed[0] === 'GET_HISTORY') { - // parsed[1] is the channel id - // parsed[2] is a validation key or an object containing metadata (optionnal) - // parsed[3] is the last known hash (optionnal) - sendMsg(ctx, user, [seq, 'ACK']); - channelName = parsed[1]; - var config = parsed[2]; - var metadata = {}; - var lastKnownHash; - - // clients can optionally pass a map of attributes - // if the channel already exists this map will be ignored - // otherwise it will be stored as the initial metadata state for the channel - if (config && typeof config === "object" && !Array.isArray(parsed[2])) { - lastKnownHash = config.lastKnownHash; - metadata = config.metadata || {}; - if (metadata.expire) { - metadata.expire = +metadata.expire * 1000 + (+new Date()); - } - } - metadata.channel = channelName; - metadata.created = +new Date(); - - // if the user sends us an invalid key, we won't be able to validate their messages - // so they'll never get written to the log anyway. Let's just drop their message - // on the floor instead of doing a bunch of extra work - // TODO send them an error message so they know something is wrong - if (metadata.validateKey && !isValidValidateKeyString(metadata.validateKey)) { - return void Log.error('HK_INVALID_KEY', metadata.validateKey); + const handleGetHistory = function (ctx, seq, user, parsed) { + // parsed[1] is the channel id + // parsed[2] is a validation key or an object containing metadata (optionnal) + // parsed[3] is the last known hash (optionnal) + sendMsg(ctx, user, [seq, 'ACK']); + var channelName = parsed[1]; + var config = parsed[2]; + var metadata = {}; + var lastKnownHash; + + // clients can optionally pass a map of attributes + // if the channel already exists this map will be ignored + // otherwise it will be stored as the initial metadata state for the channel + if (config && typeof config === "object" && !Array.isArray(parsed[2])) { + lastKnownHash = config.lastKnownHash; + metadata = config.metadata || {}; + if (metadata.expire) { + metadata.expire = +metadata.expire * 1000 + (+new Date()); } + } + metadata.channel = channelName; + metadata.created = +new Date(); + + // if the user sends us an invalid key, we won't be able to validate their messages + // so they'll never get written to the log anyway. Let's just drop their message + // on the floor instead of doing a bunch of extra work + // TODO send them an error message so they know something is wrong + if (metadata.validateKey && !isValidValidateKeyString(metadata.validateKey)) { + return void Log.error('HK_INVALID_KEY', metadata.validateKey); + } - nThen(function (waitFor) { - var w = waitFor(); + nThen(function (waitFor) { + var w = waitFor(); - /* unless this is a young channel, we will serve all messages from an offset - this will not include the channel metadata, so we need to explicitly fetch that. - unfortunately, we can't just serve it blindly, since then young channels will - send the metadata twice, so let's do a quick check of what we're going to serve... + /* unless this is a young channel, we will serve all messages from an offset + this will not include the channel metadata, so we need to explicitly fetch that. + unfortunately, we can't just serve it blindly, since then young channels will + send the metadata twice, so let's do a quick check of what we're going to serve... + */ + getIndex(ctx, channelName, waitFor((err, index) => { + /* if there's an error here, it should be encountered + and handled by the next nThen block. + so, let's just fall through... */ - getIndex(ctx, channelName, waitFor((err, index) => { - /* if there's an error here, it should be encountered - and handled by the next nThen block. - so, let's just fall through... - */ - if (err) { return w(); } - - - // it's possible that the channel doesn't have metadata - // but in that case there's no point in checking if the channel expired - // or in trying to send metadata, so just skip this block - if (!index || !index.metadata) { return void w(); } - // And then check if the channel is expired. If it is, send the error and abort - // FIXME this is hard to read because 'checkExpired' has side effects - if (checkExpired(ctx, channelName)) { return void waitFor.abort(); } - // always send metadata with GET_HISTORY requests - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); - })); - }).nThen(() => { - let msgCount = 0; - - // TODO compute lastKnownHash in a manner such that it will always skip past the metadata line? - getHistoryAsync(ctx, channelName, lastKnownHash, false, (msg, readMore) => { - if (!msg) { return; } - msgCount++; - // avoid sending the metadata message a second time - if (isMetadataMessage(msg) && metadata_cache[channelName]) { return readMore(); } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)], readMore); - }, (err) => { - if (err && err.code !== 'ENOENT') { - if (err.message !== 'EINVAL') { Log.error("HK_GET_HISTORY", err); } - const parsedMsg = {error:err.message, channel: channelName}; - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); - return; - } - - const chan = ctx.channels[channelName]; + if (err) { return w(); } + + + // it's possible that the channel doesn't have metadata + // but in that case there's no point in checking if the channel expired + // or in trying to send metadata, so just skip this block + if (!index || !index.metadata) { return void w(); } + // And then check if the channel is expired. If it is, send the error and abort + // FIXME this is hard to read because 'checkExpired' has side effects + if (checkExpired(ctx, channelName)) { return void waitFor.abort(); } + // always send metadata with GET_HISTORY requests + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); + })); + }).nThen(() => { + let msgCount = 0; - if (msgCount === 0 && !metadata_cache[channelName] && chan && chan.indexOf(user) > -1) { - metadata_cache[channelName] = metadata; + // TODO compute lastKnownHash in a manner such that it will always skip past the metadata line? + getHistoryAsync(ctx, channelName, lastKnownHash, false, (msg, readMore) => { + if (!msg) { return; } + msgCount++; + // avoid sending the metadata message a second time + if (isMetadataMessage(msg) && metadata_cache[channelName]) { return readMore(); } + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)], readMore); + }, (err) => { + if (err && err.code !== 'ENOENT') { + if (err.message !== 'EINVAL') { Log.error("HK_GET_HISTORY", err); } + const parsedMsg = {error:err.message, channel: channelName}; + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + return; + } - // the index will have already been constructed and cached at this point - // but it will not have detected any metadata because it hasn't been written yet - // this means that the cache starts off as invalid, so we have to correct it - if (chan && chan.index) { chan.index.metadata = metadata; } + const chan = ctx.channels[channelName]; + + if (msgCount === 0 && !metadata_cache[channelName] && chan && chan.indexOf(user) > -1) { + metadata_cache[channelName] = metadata; + + // the index will have already been constructed and cached at this point + // but it will not have detected any metadata because it hasn't been written yet + // this means that the cache starts off as invalid, so we have to correct it + if (chan && chan.index) { chan.index.metadata = metadata; } + + // new channels will always have their metadata written to a dedicated metadata log + // but any lines after the first which are not amendments in a particular format will be ignored. + // Thus we should be safe from race conditions here if just write metadata to the log as below... + // TODO validate this logic + // otherwise maybe we need to check that the metadata log is empty as well + store.writeMetadata(channelName, JSON.stringify(metadata), function (err) { + if (err) { + // FIXME tell the user that there was a channel error? + return void Log.error('HK_WRITE_METADATA', { + channel: channelName, + error: err, + }); + } + }); - // new channels will always have their metadata written to a dedicated metadata log - // but any lines after the first which are not amendments in a particular format will be ignored. - // Thus we should be safe from race conditions here if just write metadata to the log as below... - // TODO validate this logic - // otherwise maybe we need to check that the metadata log is empty as well - store.writeMetadata(channelName, JSON.stringify(metadata), function (err) { + // write tasks + if(tasks && metadata.expire && typeof(metadata.expire) === 'number') { + // the fun part... + // the user has said they want this pad to expire at some point + tasks.write(metadata.expire, "EXPIRE", [ channelName ], function (err) { if (err) { - // FIXME tell the user that there was a channel error? - return void Log.error('HK_WRITE_METADATA', { - channel: channelName, - error: err, - }); + // if there is an error, we don't want to crash the whole server... + // just log it, and if there's a problem you'll be able to fix it + // at a later date with the provided information + Log.error('HK_CREATE_EXPIRE_TASK', err); + Log.info('HK_INVALID_EXPIRE_TASK', JSON.stringify([metadata.expire, 'EXPIRE', channelName])); } }); - - // write tasks - if(tasks && metadata.expire && typeof(metadata.expire) === 'number') { - // the fun part... - // the user has said they want this pad to expire at some point - tasks.write(metadata.expire, "EXPIRE", [ channelName ], function (err) { - if (err) { - // if there is an error, we don't want to crash the whole server... - // just log it, and if there's a problem you'll be able to fix it - // at a later date with the provided information - Log.error('HK_CREATE_EXPIRE_TASK', err); - Log.info('HK_INVALID_EXPIRE_TASK', JSON.stringify([metadata.expire, 'EXPIRE', channelName])); - } - }); - } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(metadata)]); } + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(metadata)]); + } - // End of history message: - let parsedMsg = {state: 1, channel: channelName}; - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); - }); + // End of history message: + let parsedMsg = {state: 1, channel: channelName}; + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); }); - } else if (parsed[0] === 'GET_HISTORY_RANGE') { - channelName = parsed[1]; - var map = parsed[2]; - if (!(map && typeof(map) === 'object')) { - return void sendMsg(ctx, user, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]); - } + }); + }; - var oldestKnownHash = map.from; - var desiredMessages = map.count; - var desiredCheckpoint = map.cpCount; - var txid = map.txid; - if (typeof(desiredMessages) !== 'number' && typeof(desiredCheckpoint) !== 'number') { - return void sendMsg(ctx, user, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]); - } + const handleGetHistoryRange = function (ctx, seq, user, parsed) { + var channelName = parsed[1]; + var map = parsed[2]; + if (!(map && typeof(map) === 'object')) { + return void sendMsg(ctx, user, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]); + } - if (!txid) { - return void sendMsg(ctx, user, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]); - } + var oldestKnownHash = map.from; + var desiredMessages = map.count; + var desiredCheckpoint = map.cpCount; + var txid = map.txid; + if (typeof(desiredMessages) !== 'number' && typeof(desiredCheckpoint) !== 'number') { + return void sendMsg(ctx, user, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]); + } - sendMsg(ctx, user, [seq, 'ACK']); - return void getOlderHistory(channelName, oldestKnownHash, function (messages) { - var toSend = []; - if (typeof (desiredMessages) === "number") { - toSend = messages.slice(-desiredMessages); - } else { - let cpCount = 0; - for (var i = messages.length - 1; i >= 0; i--) { - if (/^cp\|/.test(messages[i][4]) && i !== (messages.length - 1)) { - cpCount++; - } - toSend.unshift(messages[i]); - if (cpCount >= desiredCheckpoint) { break; } + if (!txid) { + return void sendMsg(ctx, user, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]); + } + + sendMsg(ctx, user, [seq, 'ACK']); + return void getOlderHistory(channelName, oldestKnownHash, function (messages) { + var toSend = []; + if (typeof (desiredMessages) === "number") { + toSend = messages.slice(-desiredMessages); + } else { + let cpCount = 0; + for (var i = messages.length - 1; i >= 0; i--) { + if (/^cp\|/.test(messages[i][4]) && i !== (messages.length - 1)) { + cpCount++; } + toSend.unshift(messages[i]); + if (cpCount >= desiredCheckpoint) { break; } } - toSend.forEach(function (msg) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, - JSON.stringify(['HISTORY_RANGE', txid, msg])]); - }); - + } + toSend.forEach(function (msg) { sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, - JSON.stringify(['HISTORY_RANGE_END', txid, channelName]) - ]); - }); - } else if (parsed[0] === 'GET_FULL_HISTORY') { - // parsed[1] is the channel id - // parsed[2] is a validation key (optionnal) - // parsed[3] is the last known hash (optionnal) - sendMsg(ctx, user, [seq, 'ACK']); - - // FIXME should we send metadata here too? - // none of the clientside code which uses this API needs metadata, but it won't hurt to send it (2019-08-22) - getHistoryAsync(ctx, parsed[1], -1, false, (msg, readMore) => { - if (!msg) { return; } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['FULL_HISTORY', msg])], readMore); - }, (err) => { - let parsedMsg = ['FULL_HISTORY_END', parsed[1]]; - if (err) { - Log.error('HK_GET_FULL_HISTORY', err.stack); - parsedMsg = ['ERROR', parsed[1], err.message]; - } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + JSON.stringify(['HISTORY_RANGE', txid, msg])]); }); - } else if (rpc) { - /* RPC Calls... */ - var rpc_call = parsed.slice(1); - sendMsg(ctx, user, [seq, 'ACK']); - try { + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, + JSON.stringify(['HISTORY_RANGE_END', txid, channelName]) + ]); + }); + }; + + const handleGetFullHistory = function (ctx, seq, user, parsed) { + // parsed[1] is the channel id + // parsed[2] is a validation key (optionnal) + // parsed[3] is the last known hash (optionnal) + sendMsg(ctx, user, [seq, 'ACK']); + + // FIXME should we send metadata here too? + // none of the clientside code which uses this API needs metadata, but it won't hurt to send it (2019-08-22) + return void getHistoryAsync(ctx, parsed[1], -1, false, (msg, readMore) => { + if (!msg) { return; } + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['FULL_HISTORY', msg])], readMore); + }, (err) => { + let parsedMsg = ['FULL_HISTORY_END', parsed[1]]; + if (err) { + Log.error('HK_GET_FULL_HISTORY', err.stack); + parsedMsg = ['ERROR', parsed[1], err.message]; + } + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + }); + }; + + const handleRPC = function (ctx, seq, user, parsed) { + if (typeof(rpc) !== 'function') { return; } + + /* RPC Calls... */ + var rpc_call = parsed.slice(1); + + sendMsg(ctx, user, [seq, 'ACK']); + try { // slice off the sequence number and pass in the rest of the message rpc(ctx, rpc_call, function (err, output) { if (err) { @@ -992,10 +969,43 @@ module.exports.create = function (cfg) { // finally, send a response to the client that sent the RPC sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0]].concat(output))]); }); - } catch (e) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]); - } + } catch (e) { + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]); + } + }; + + /* onDirectMessage + * exported for use by the netflux-server + * parses and handles all direct messages directed to the history keeper + * check if it's expired and execute all the associated side-effects + * routes queries to the appropriate handlers + */ + const onDirectMessage = function (ctx, seq, user, json) { + Log.silly('HK_MESSAGE', json); + + let parsed; + try { + parsed = JSON.parse(json[2]); + } catch (err) { + Log.error("HK_PARSE_CLIENT_MESSAGE", json); + return; + } + + // If the requested history is for an expired channel, abort + // Note the if we don't have the keys for that channel in metadata_cache, we'll + // have to abort later (once we know the expiration time) + if (checkExpired(ctx, parsed[1])) { return; } + + if (parsed[0] === 'GET_HISTORY') { + return void handleGetHistory(ctx, seq, user, parsed); + } + if (parsed[0] === 'GET_HISTORY_RANGE') { + return void handleGetHistoryRange(ctx, seq, user, parsed); + } + if (parsed[0] === 'GET_FULL_HISTORY') { + return void handleGetFullHistory(ctx, seq, user, parsed); } + return void handleRPC(ctx, seq, user, parsed); }; return { From e3f5c893336ffa4a2cf02f9935c802da6c084d66 Mon Sep 17 00:00:00 2001 From: yflory Date: Thu, 23 Jan 2020 16:11:06 +0100 Subject: [PATCH 009/172] Remove window.location.hash and window.location.href from common-hash --- www/common/common-hash.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/www/common/common-hash.js b/www/common/common-hash.js index 85ec3b36e..90ccf805c 100644 --- a/www/common/common-hash.js +++ b/www/common/common-hash.js @@ -198,7 +198,13 @@ Version 1 parsed.version = 2; parsed.app = hashArr[2]; parsed.mode = hashArr[3]; - parsed.key = hashArr[4]; + + // Check if the key is a channel ID + if (/^[a-f0-9]{32,34}$/.test(hashArr[4])) { + parsed.channel = hashArr[4]; + } else { + parsed.key = hashArr[4]; + } options = hashArr.slice(5); parsed.password = options.indexOf('p') !== -1; @@ -345,7 +351,7 @@ Version 1 secret.version = 2; secret.type = type; }; - if (!secretHash && !window.location.hash) { //!/#/.test(window.location.href)) { + if (!secretHash) { generate(); return secret; } else { @@ -355,12 +361,7 @@ Version 1 if (!type) { throw new Error("getSecrets with a hash requires a type parameter"); } parsed = parseTypeHash(type, secretHash); hash = secretHash; - } else { - var pHref = parsePadUrl(window.location.href); - parsed = pHref.hashData; - hash = pHref.hash; } - //var hash = secretHash || window.location.hash.slice(1); if (hash.length === 0) { generate(); return secret; From b585dd998dc255126128d6fe4bdc73a0409d6ca0 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 13:27:12 -0500 Subject: [PATCH 010/172] throw in a little asynchrony --- lib/schedule.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/schedule.js b/lib/schedule.js index 1e93a2126..1fdef8cce 100644 --- a/lib/schedule.js +++ b/lib/schedule.js @@ -68,7 +68,6 @@ var isEmpty = function (map) { return true; }; -// XXX enforce asynchrony everywhere module.exports = function () { // every scheduler instance has its own queue var queue = WriteQueue(); @@ -111,7 +110,7 @@ module.exports = function () { if (typeof(local.waiting) !== 'function') { return void collectGarbage(); } - local.waiting(); + setTimeout(local.waiting); }); }; From 75f1f8c40b63af8f3cee1113e815e4ea06780cf5 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 13:56:16 -0500 Subject: [PATCH 011/172] refactor some methods that will be used --- rpc.js | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/rpc.js b/rpc.js index 8041cdcad..0a2c5b17f 100644 --- a/rpc.js +++ b/rpc.js @@ -278,6 +278,24 @@ var getMetadata = function (Env, channel, cb) { }); }; +// E_NO_OWNERS +var hasOwners = function (metadata) { + return Boolean(metadata && Array.isArray(metadata.owners)); +}; + +var hasPendingOwners = function (metadata) { + return Boolean(metadata && Array.isArray(metadata.pending_owners)); +}; + +// INSUFFICIENT_PERMISSIONS +var isOwner = function (metadata, unsafeKey) { + return metadata.owners.indexOf(unsafeKey) !== -1; +}; + +var isPendingOwner = function (metadata, unsafeKey) { + return metadata.pending_owners.indexOf(unsafeKey) !== -1; +}; + /* setMetadata - write a new line to the metadata log if a valid command is provided - data is an object: { @@ -300,16 +318,16 @@ var setMetadata = function (Env, data, unsafeKey, cb) { cb(err); return void next(); } - if (!(metadata && Array.isArray(metadata.owners))) { + if (!hasOwners(metadata)) { cb('E_NO_OWNERS'); return void next(); } // Confirm that the channel is owned by the user in question - // or the user is accepting a pending ownerhsip offer - if (metadata.pending_owners && Array.isArray(metadata.pending_owners) && - metadata.pending_owners.indexOf(unsafeKey) !== -1 && - metadata.owners.indexOf(unsafeKey) === -1) { + // or the user is accepting a pending ownership offer + if (hasPendingOwners(metadata) && + isPendingOwner(metadata, unsafeKey) && + !isOwner(metadata, unsafeKey)) { // If you are a pending owner, make sure you can only add yourelf as an owner if ((command !== 'ADD_OWNERS' && command !== 'RM_PENDING_OWNERS') @@ -319,8 +337,8 @@ var setMetadata = function (Env, data, unsafeKey, cb) { cb('INSUFFICIENT_PERMISSIONS'); return void next(); } - - } else if (metadata.owners.indexOf(unsafeKey) === -1) { + // XXX wacky fallthrough is hard to read + } else if (!isOwner(metadata, unsafeKey)) { cb('INSUFFICIENT_PERMISSIONS'); return void next(); } @@ -817,12 +835,11 @@ var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } - if (!(metadata && Array.isArray(metadata.owners))) { return void cb('E_NO_OWNERS'); } + if (!hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } // Confirm that the channel is owned by the user in question - if (metadata.owners.indexOf(unsafeKey) === -1) { + if (!isOwner(metadata, unsafeKey)) { return void cb('INSUFFICIENT_PERMISSIONS'); } - // FIXME COLDSTORAGE return void Env.msgStore.clearChannel(channelId, function (e) { cb(e); }); @@ -905,8 +922,8 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } - if (!(metadata && Array.isArray(metadata.owners))) { return void cb('E_NO_OWNERS'); } - if (metadata.owners.indexOf(unsafeKey) === -1) { + if (!hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } + if (!isOwner(metadata, unsafeKey)) { return void cb('INSUFFICIENT_PERMISSIONS'); } // if the admin has configured data retention... From 47290fca1e60fc254d52214c2343f6772c2dec2f Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 14:10:20 -0500 Subject: [PATCH 012/172] leave some notes about something that was tricky to read --- rpc.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/rpc.js b/rpc.js index 0a2c5b17f..69eb9313f 100644 --- a/rpc.js +++ b/rpc.js @@ -323,6 +323,14 @@ var setMetadata = function (Env, data, unsafeKey, cb) { return void next(); } + // if you are a pending owner and not an owner + // you can either ADD_OWNERS, or RM_PENDING_OWNERS + // and you should only be able to add yourself as an owner + // everything else should be rejected + // else if you are not an owner + // you should be rejected + // else write the command + // Confirm that the channel is owned by the user in question // or the user is accepting a pending ownership offer if (hasPendingOwners(metadata) && @@ -337,7 +345,9 @@ var setMetadata = function (Env, data, unsafeKey, cb) { cb('INSUFFICIENT_PERMISSIONS'); return void next(); } - // XXX wacky fallthrough is hard to read + // FIXME wacky fallthrough is hard to read + // we could pass this off to a writeMetadataCommand function + // and make the flow easier to follow } else if (!isOwner(metadata, unsafeKey)) { cb('INSUFFICIENT_PERMISSIONS'); return void next(); From 88dcadcb1b561bfc6297b7bb7aad20a3f785e925 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 14:31:45 -0500 Subject: [PATCH 013/172] sketch out trimHistory interface --- rpc.js | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/rpc.js b/rpc.js index 69eb9313f..f690b2bf1 100644 --- a/rpc.js +++ b/rpc.js @@ -961,13 +961,27 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { }; var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { - // XXX validate that the user sending the request owns the channel in question - // proceed to call Env.msgStore.trimChannel(channelId, hash, cb) if ok - // otherwise reject with INSUFFICIENT_PERMISSIONS + nThen(function (w) { + getMetadata(Env, channelId, w(function (err, metadata) { + if (err) { return void cb(err); } + if (!hasOwners(metadata)) { + w.abort(); + return void cb('E_NO_OWNERS'); + } + if (!isOwner(metadata, unsafeKey)) { + w.abort(); + return void cb("INSUFFICIENT_PERMISSIONS"); + } + // else fall through to the next block + })); + }).nThen(function () { + Env.msgStore.trimChannel(channelId, hash, function (err) { + if (err) { return void cb(err); } + - // XXX if trimChannel calls back without an error - // you must also clear the channel's index from historyKeeper cache - cb("E_NOT_IMPLEMENTED"); + // XXX you must also clear the channel's index from historyKeeper cache + }); + }); }; /* Users should be able to clear their own pin log with an authenticated RPC From 873a7c7c84b01e77670a821e0712b285f543eba9 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 14:32:10 -0500 Subject: [PATCH 014/172] remove some flow annotations --- rpc.js | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/rpc.js b/rpc.js index f690b2bf1..948977bbd 100644 --- a/rpc.js +++ b/rpc.js @@ -1411,24 +1411,7 @@ var upload_status = function (Env, safeKey, filesize, _cb) { // FIXME FILES }); }; -/*:: -const flow_Config = require('./config.example.js'); -type Config_t = typeof(flow_Config); -import type { ChainPadServer_Storage_t } from './storage/file.js' -type NetfluxWebsocketSrvContext_t = { - store: ChainPadServer_Storage_t, - getHistoryOffset: ( - ctx: NetfluxWebsocketSrvContext_t, - channelName: string, - lastKnownHash: ?string, - cb: (err: ?Error, offset: ?number)=>void - )=>void -}; -*/ -RPC.create = function ( - config /*:Config_t*/, - cb /*:(?Error, ?Function)=>void*/ -) { +RPC.create = function (config, cb) { Log = config.log; // load pin-store... @@ -1445,7 +1428,7 @@ RPC.create = function ( Sessions: {}, paths: {}, msgStore: config.store, - pinStore: (undefined /*:any*/), + pinStore: undefined, pinnedPads: {}, evPinnedPadsReady: mkEvent(true), limits: {}, @@ -1782,11 +1765,7 @@ RPC.create = function ( handleMessage(true); }; - var rpc = function ( - ctx /*:NetfluxWebsocketSrvContext_t*/, - data /*:Array>*/, - respond /*:(?string, ?Array)=>void*/) - { + var rpc = function (ctx, data, respond) { try { return rpc0(ctx, data, respond); } catch (e) { From c14612528394d67fad427907d2f956a979181dba Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 14:32:32 -0500 Subject: [PATCH 015/172] sketch out a method for graceful shutdown --- rpc.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/rpc.js b/rpc.js index 948977bbd..1790682ec 100644 --- a/rpc.js +++ b/rpc.js @@ -1318,6 +1318,16 @@ var getActiveSessions = function (Env, ctx, cb) { cb (void 0, [total, ips.length]); }; +var shutdown = function (Env, ctx, cb) { + return void cb('E_NOT_IMPLEMENTED'); + //clearInterval(Env.sessionExpirationInterval); + // XXX set a flag to prevent incoming database writes + // XXX disconnect all users and reject new connections + // XXX wait until all pending writes are complete + // then process.exit(0); + // and allow system functionality to restart the server +}; + var adminCommand = function (Env, ctx, publicKey, config, data, cb) { var admins = Env.admins; if (admins.indexOf(publicKey) === -1) { @@ -1336,6 +1346,8 @@ var adminCommand = function (Env, ctx, publicKey, config, data, cb) { case 'FLUSH_CACHE': config.flushCache(); return cb(void 0, true); + case 'SHUTDOWN': + return shutdown(Env, ctx, cb); default: return cb('UNHANDLED_ADMIN_COMMAND'); } @@ -1806,7 +1818,8 @@ RPC.create = function (config, cb) { }).nThen(function () { cb(void 0, rpc); // expire old sessions once per minute - setInterval(function () { + // XXX allow for graceful shutdown + Env.sessionExpirationInterval = setInterval(function () { expireSessions(Sessions); }, SESSION_EXPIRATION_TIME); }); From c4194117a7896f7d968c745dbfaf7cb867b28d21 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 14:33:00 -0500 Subject: [PATCH 016/172] ever so slightly faster direct message handler --- historyKeeper.js | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/historyKeeper.js b/historyKeeper.js index fe16a204c..2cb0f98ec 100644 --- a/historyKeeper.js +++ b/historyKeeper.js @@ -974,6 +974,12 @@ module.exports.create = function (cfg) { } }; + const directMessageCommands = { + GET_HISTORY: handleGetHistory, + GET_HISTORY_RANGE: handleGetHistoryRange, + GET_FULL_HISTORY: handleGetFullHistory, + }; + /* onDirectMessage * exported for use by the netflux-server * parses and handles all direct messages directed to the history keeper @@ -996,16 +1002,11 @@ module.exports.create = function (cfg) { // have to abort later (once we know the expiration time) if (checkExpired(ctx, parsed[1])) { return; } - if (parsed[0] === 'GET_HISTORY') { - return void handleGetHistory(ctx, seq, user, parsed); - } - if (parsed[0] === 'GET_HISTORY_RANGE') { - return void handleGetHistoryRange(ctx, seq, user, parsed); - } - if (parsed[0] === 'GET_FULL_HISTORY') { - return void handleGetFullHistory(ctx, seq, user, parsed); - } - return void handleRPC(ctx, seq, user, parsed); + // look up the appropriate command in the map of commands or fall back to RPC + var command = directMessageCommands[parsed[0]] || handleRPC; + + // run the command with the standard function signature + command(ctx, seq, user, parsed); }; return { From f45de2b52f81465c2d3293512216aaf06f38f178 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 14:47:55 -0500 Subject: [PATCH 017/172] move some server deps from repo root to lib/ --- historyKeeper.js => lib/historyKeeper.js | 8 ++++---- rpc.js => lib/rpc.js | 22 +++++++++++++--------- server.js | 4 ++-- 3 files changed, 19 insertions(+), 15 deletions(-) rename historyKeeper.js => lib/historyKeeper.js (99%) rename rpc.js => lib/rpc.js (98%) diff --git a/historyKeeper.js b/lib/historyKeeper.js similarity index 99% rename from historyKeeper.js rename to lib/historyKeeper.js index 2cb0f98ec..414d064c3 100644 --- a/historyKeeper.js +++ b/lib/historyKeeper.js @@ -5,10 +5,10 @@ const nThen = require('nthen'); const Nacl = require('tweetnacl/nacl-fast'); const Crypto = require('crypto'); -const Once = require("./lib/once"); -const Meta = require("./lib/metadata"); -const WriteQueue = require("./lib/write-queue"); -const BatchRead = require("./lib/batch-read"); +const Once = require("./once"); +const Meta = require("./metadata"); +const WriteQueue = require("./write-queue"); +const BatchRead = require("./batch-read"); let Log; const now = function () { return (new Date()).getTime(); }; diff --git a/rpc.js b/lib/rpc.js similarity index 98% rename from rpc.js rename to lib/rpc.js index 1790682ec..dfe27302d 100644 --- a/rpc.js +++ b/lib/rpc.js @@ -11,25 +11,25 @@ var Fs = require("fs"); var Fse = require("fs-extra"); var Path = require("path"); var Https = require("https"); -const Package = require('./package.json'); -const Pinned = require('./scripts/pinned'); +const Package = require('../package.json'); +const Pinned = require('../scripts/pinned'); const Saferphore = require("saferphore"); const nThen = require("nthen"); const getFolderSize = require("get-folder-size"); -const Pins = require("./lib/pins"); -const Meta = require("./lib/metadata"); -const WriteQueue = require("./lib/write-queue"); -const BatchRead = require("./lib/batch-read"); +const Pins = require("./pins"); +const Meta = require("./metadata"); +const WriteQueue = require("./write-queue"); +const BatchRead = require("./batch-read"); -const Util = require("./lib/common-util"); +const Util = require("./common-util"); const escapeKeyCharacters = Util.escapeKeyCharacters; const unescapeKeyCharacters = Util.unescapeKeyCharacters; const mkEvent = Util.mkEvent; var RPC = module.exports; -var Store = require("./storage/file"); -var BlobStore = require("./storage/blob"); +var Store = require("../storage/file"); +var BlobStore = require("../storage/blob"); var DEFAULT_LIMIT = 50 * 1024 * 1024; var SESSION_EXPIRATION_TIME = 60 * 1000; @@ -611,6 +611,9 @@ var updateLimits = function (Env, config, publicKey, cb /*:(?string, ?any[])=>vo req.end(body); }; +// XXX it's possible for this to respond before the server has had a chance +// to fetch the limits. Maybe we should respond with an error... +// or wait until we actually know the limits before responding var getLimit = function (Env, publicKey, cb) { var unescapedKey = unescapeKeyCharacters(publicKey); var limit = Env.limits[unescapedKey]; @@ -1445,6 +1448,7 @@ RPC.create = function (config, cb) { evPinnedPadsReady: mkEvent(true), limits: {}, admins: [], + sessionExpirationInterval: undefined, }; try { diff --git a/server.js b/server.js index 399eb1442..fec59a7ea 100644 --- a/server.js +++ b/server.js @@ -264,7 +264,7 @@ var nt = nThen(function (w) { }, 1000 * 60 * 5); // run every five minutes })); }).nThen(function (w) { - require("./rpc").create(config, w(function (e, _rpc) { + require("./lib/rpc").create(config, w(function (e, _rpc) { if (e) { w.abort(); throw e; @@ -272,7 +272,7 @@ var nt = nThen(function (w) { rpc = _rpc; })); }).nThen(function () { - var HK = require('./historyKeeper.js'); + var HK = require('./lib/historyKeeper.js'); var hkConfig = { tasks: config.tasks, rpc: rpc, From c388641479128303363d8a4247f64230c08a7264 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 15:13:19 -0500 Subject: [PATCH 018/172] drop support for 'retainData' configuration --- config/config.example.js | 21 +++++---------- lib/historyKeeper.js | 16 +++-------- lib/rpc.js | 57 ++++++--------------------------------- scripts/evict-inactive.js | 50 ++++------------------------------ server.js | 1 - storage/file.js | 4 --- storage/tasks.js | 17 ------------ 7 files changed, 22 insertions(+), 144 deletions(-) diff --git a/config/config.example.js b/config/config.example.js index 9981c0626..90e96a66a 100644 --- a/config/config.example.js +++ b/config/config.example.js @@ -231,26 +231,17 @@ module.exports = { */ inactiveTime: 90, // days - /* CryptPad can be configured to remove inactive data which has not been pinned. - * Deletion of data is always risky and as an operator you have the choice to - * archive data instead of deleting it outright. Set this value to true if - * you want your server to archive files and false if you want to keep using - * the old behaviour of simply removing files. + /* CryptPad archives some data instead of deleting it outright. + * This archived data still takes up space and so you'll probably still want to + * remove these files after a brief period. + * + * cryptpad/scripts/evict-inactive.js is intended to be run daily + * from a crontab or similar scheduling service. * - * WARNING: this is not implemented universally, so at the moment this will - * only apply to the removal of 'channels' due to inactivity. - */ - retainData: true, - - /* As described above, CryptPad offers the ability to archive some data - * instead of deleting it outright. This archived data still takes up space - * and so you'll probably still want to remove these files after a brief period. * The intent with this feature is to provide a safety net in case of accidental * deletion. Set this value to the number of days you'd like to retain * archived data before it's removed permanently. * - * If 'retainData' is set to false, there will never be any archived data - * to remove. */ archiveRetentionTime: 15, diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 414d064c3..09f5fdfea 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -83,7 +83,6 @@ module.exports.create = function (cfg) { const rpc = cfg.rpc; const tasks = cfg.tasks; const store = cfg.store; - const retainData = cfg.retainData; Log = cfg.log; Log.silly('HK_LOADING', 'LOADING HISTORY_KEEPER MODULE'); @@ -350,18 +349,9 @@ module.exports.create = function (cfg) { but for some reason are still present */ const expireChannel = function (ctx, channel) { - if (retainData) { - return void store.archiveChannel(channel, function (err) { - Log.info("ARCHIVAL_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", { - channelId: channel, - status: err? String(err): "SUCCESS", - }); - }); - } - - store.removeChannel(channel, function (err) { - Log.info("DELETION_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", { - channelid: channel, + return void store.archiveChannel(channel, function (err) { + Log.info("ARCHIVAL_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", { + channelId: channel, status: err? String(err): "SUCCESS", }); }); diff --git a/lib/rpc.js b/lib/rpc.js index dfe27302d..0ece7915c 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -878,22 +878,8 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { })); }).nThen(function (w) { // remove the blob - - if (Env.retainData) { - return void Env.blobStore.archive.blob(blobId, w(function (err) { - Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', { - safeKey: safeKey, - blobId: blobId, - status: err? String(err): 'SUCCESS', - }); - if (err) { - w.abort(); - return void cb(err); - } - })); - } - Env.blobStore.remove.blob(blobId, w(function (err) { - Log.info('DELETION_OWNED_FILE_BY_OWNER_RPC', { + return void Env.blobStore.archive.blob(blobId, w(function (err) { + Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', { safeKey: safeKey, blobId: blobId, status: err? String(err): 'SUCCESS', @@ -904,23 +890,9 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { } })); }).nThen(function () { - // remove the proof - if (Env.retainData) { - return void Env.blobStore.archive.proof(safeKey, blobId, function (err) { - Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", { - safeKey: safeKey, - blobId: blobId, - status: err? String(err): 'SUCCESS', - }); - if (err) { - return void cb("E_PROOF_REMOVAL"); - } - cb(); - }); - } - - Env.blobStore.remove.proof(safeKey, blobId, function (err) { - Log.info("DELETION_PROOF_REMOVAL_BY_OWNER_RPC", { + // archive the proof + return void Env.blobStore.archive.proof(safeKey, blobId, function (err) { + Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", { safeKey: safeKey, blobId: blobId, status: err? String(err): 'SUCCESS', @@ -939,21 +911,9 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { if (!isOwner(metadata, unsafeKey)) { return void cb('INSUFFICIENT_PERMISSIONS'); } - // if the admin has configured data retention... - // temporarily archive the file instead of removing it - if (Env.retainData) { - return void Env.msgStore.archiveChannel(channelId, function (e) { - Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', { - unsafeKey: unsafeKey, - channelId: channelId, - status: e? String(e): 'SUCCESS', - }); - cb(e); - }); - } - - return void Env.msgStore.removeChannel(channelId, function (e) { - Log.info('DELETION_CHANNEL_BY_OWNER_RPC', { + // temporarily archive the file + return void Env.msgStore.archiveChannel(channelId, function (e) { + Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', { unsafeKey: unsafeKey, channelId: channelId, status: e? String(e): 'SUCCESS', @@ -1437,7 +1397,6 @@ RPC.create = function (config, cb) { }; var Env = { - retainData: config.retainData || false, defaultStorageLimit: config.defaultStorageLimit, maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024), Sessions: {}, diff --git a/scripts/evict-inactive.js b/scripts/evict-inactive.js index 13028b8ff..f0e801909 100644 --- a/scripts/evict-inactive.js +++ b/scripts/evict-inactive.js @@ -15,8 +15,6 @@ var inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000); // files which were archived before this date can be considered safe to remove var retentionTime = +new Date() - (config.archiveRetentionTime * 24 * 3600 * 1000); -var retainData = Boolean(config.retainData); - var getNewestTime = function (stats) { return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) { return stats[b] > stats[a]? b: a; @@ -176,23 +174,6 @@ nThen(function (w) { if (pins[item.blobId]) { return void next(); } if (item && getNewestTime(item) > retentionTime) { return void next(); } - if (!retainData) { - return void blobs.remove.blob(item.blobId, function (err) { - if (err) { - Log.error("EVICT_BLOB_ERROR", { - error: err, - item: item, - }); - return void next(); - } - Log.info("EVICT_BLOB_INACTIVE", { - item: item, - }); - removed++; - next(); - }); - } - blobs.archive.blob(item.blobId, function (err) { if (err) { Log.error("EVICT_ARCHIVE_BLOB_ERROR", { @@ -247,7 +228,6 @@ nThen(function (w) { Log.info("EVICT_BLOB_PROOFS_REMOVED", removed); })); }).nThen(function (w) { - var removed = 0; var channels = 0; var archived = 0; @@ -279,42 +259,22 @@ nThen(function (w) { // ignore the channel if it's pinned if (pins[item.channel]) { return void cb(); } - // if the server is configured to retain data, archive the channel - if (config.retainData) { - return void store.archiveChannel(item.channel, w(function (err) { - if (err) { - Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', { - error: err, - channel: item.channel, - }); - return void cb(); - } - Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel); - archived++; - cb(); - })); - } - - // otherwise remove it - store.removeChannel(item.channel, w(function (err) { + return void store.archiveChannel(item.channel, w(function (err) { if (err) { - Log.error('EVICT_CHANNEL_REMOVAL_ERROR', { + Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', { error: err, channel: item.channel, }); return void cb(); } - Log.info('EVICT_CHANNEL_REMOVAL', item.channel); - removed++; + Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel); + archived++; cb(); })); }; var done = function () { - if (config.retainData) { - return void Log.info('EVICT_CHANNELS_ARCHIVED', archived); - } - return void Log.info('EVICT_CHANNELS_REMOVED', removed); + return void Log.info('EVICT_CHANNELS_ARCHIVED', archived); }; store.listChannels(handler, w(done)); diff --git a/server.js b/server.js index fec59a7ea..d2bdcabd8 100644 --- a/server.js +++ b/server.js @@ -278,7 +278,6 @@ var nt = nThen(function (w) { rpc: rpc, store: config.store, log: log, - retainData: Boolean(config.retainData), }; historyKeeper = HK.create(hkConfig); }).nThen(function () { diff --git a/storage/file.js b/storage/file.js index b2899619f..a3d6f521f 100644 --- a/storage/file.js +++ b/storage/file.js @@ -555,9 +555,6 @@ var listChannels = function (root, handler, cb) { // to an equivalent location in the cold storage directory var archiveChannel = function (env, channelName, cb) { // TODO close channels before archiving them? - if (!env.retainData) { - return void cb("ARCHIVES_DISABLED"); - } // ctime is the most reliable indicator of when a file was archived // because it is used to indicate changes to the files metadata @@ -875,7 +872,6 @@ module.exports.create = function (conf, cb) { var env = { root: conf.filePath || './datastore', archiveRoot: conf.archivePath || './data/archive', - retainData: conf.retainData, channels: { }, channelExpirationMs: conf.channelExpirationMs || 30000, verbose: conf.verbose, diff --git a/storage/tasks.js b/storage/tasks.js index 2209b3d59..bb4dbdb9c 100644 --- a/storage/tasks.js +++ b/storage/tasks.js @@ -202,22 +202,6 @@ var expire = function (env, task, cb) { var Log = env.log; var args = task.slice(2); - if (!env.retainData) { - Log.info('DELETION_SCHEDULED_EXPIRATION', { - task: task, - }); - env.store.removeChannel(args[0], function (err) { - if (err) { - Log.error('DELETION_SCHEDULED_EXPIRATION_ERROR', { - task: task, - error: err, - }); - } - cb(); - }); - return; - } - Log.info('ARCHIVAL_SCHEDULED_EXPIRATION', { task: task, }); @@ -381,7 +365,6 @@ Tasks.create = function (config, cb) { root: config.taskPath || './tasks', log: config.log, store: config.store, - retainData: Boolean(config.retainData), }; // make sure the path exists... From 9cdf54aff2def861b3f23a05fa97a91aa7c32149 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 17:58:24 -0500 Subject: [PATCH 019/172] untested implementation of trimHistory --- lib/historyKeeper.js | 30 ++----- lib/hk-util.js | 24 ++++++ storage/file.js | 198 +++++++++++++++++++++++++++++++++++++++---- 3 files changed, 211 insertions(+), 41 deletions(-) create mode 100644 lib/hk-util.js diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 09f5fdfea..c3ed67cb7 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -10,29 +10,13 @@ const Meta = require("./metadata"); const WriteQueue = require("./write-queue"); const BatchRead = require("./batch-read"); +const Extras = require("./hk-util.js"); + let Log; const now = function () { return (new Date()).getTime(); }; const ONE_DAY = 1000 * 60 * 60 * 24; // one day in milliseconds -/* getHash - * this function slices off the leading portion of a message which is - most likely unique - * these "hashes" are used to identify particular messages in a channel's history - * clients store "hashes" either in memory or in their drive to query for new messages: - * when reconnecting to a pad - * when connecting to chat or a mailbox - * thus, we can't change this function without invalidating client data which: - * is encrypted clientside - * can't be easily migrated - * don't break it! -*/ -const getHash = function (msg) { - if (typeof(msg) !== 'string') { - Log.warn('HK_GET_HASH', 'getHash() called on ' + typeof(msg) + ': ' + msg); - return ''; - } - return msg.slice(0,64); -}; +const getHash = Extras.getHash; const tryParse = function (str) { try { @@ -185,7 +169,7 @@ module.exports.create = function (cfg) { if (msg[0] === 0 && msg[2] === 'MSG' && typeof(msg[4]) === 'string') { // msgObj.offset is API guaranteed by our storage module // it should always be a valid positive integer - offsetByHash[getHash(msg[4])] = msgObj.offset; + offsetByHash[getHash(msg[4], Log)] = msgObj.offset; } // There is a trailing \n at the end of the file size = msgObj.offset + msgObj.buff.length + 1; @@ -502,7 +486,7 @@ module.exports.create = function (cfg) { msgStruct.push(now()); // storeMessage - storeMessage(ctx, channel, JSON.stringify(msgStruct), isCp, getHash(msgStruct[4])); + storeMessage(ctx, channel, JSON.stringify(msgStruct), isCp, getHash(msgStruct[4], Log)); }); }; @@ -601,7 +585,7 @@ module.exports.create = function (cfg) { const msg = tryParse(msgObj.buff.toString('utf8')); // if it was undefined then go onto the next message if (typeof msg === "undefined") { return readMore(); } - if (typeof(msg[4]) !== 'string' || lastKnownHash !== getHash(msg[4])) { + if (typeof(msg[4]) !== 'string' || lastKnownHash !== getHash(msg[4], Log)) { return void readMore(); } offset = msgObj.offset; @@ -672,7 +656,7 @@ module.exports.create = function (cfg) { var content = parsed[4]; if (typeof(content) !== 'string') { return; } - var hash = getHash(content); + var hash = getHash(content, Log); if (hash === oldestKnownHash) { found = true; } diff --git a/lib/hk-util.js b/lib/hk-util.js new file mode 100644 index 000000000..cea39c4e1 --- /dev/null +++ b/lib/hk-util.js @@ -0,0 +1,24 @@ +var HK = module.exports; + +/* getHash + * this function slices off the leading portion of a message which is + most likely unique + * these "hashes" are used to identify particular messages in a channel's history + * clients store "hashes" either in memory or in their drive to query for new messages: + * when reconnecting to a pad + * when connecting to chat or a mailbox + * thus, we can't change this function without invalidating client data which: + * is encrypted clientside + * can't be easily migrated + * don't break it! +*/ +HK.getHash = function (msg, Log) { + if (typeof(msg) !== 'string') { + if (Log) { + Log.warn('HK_GET_HASH', 'getHash() called on ' + typeof(msg) + ': ' + msg); + } + return ''; + } + return msg.slice(0,64); +}; + diff --git a/storage/file.js b/storage/file.js index a3d6f521f..08ec98f85 100644 --- a/storage/file.js +++ b/storage/file.js @@ -7,6 +7,8 @@ var Path = require("path"); var nThen = require("nthen"); var Semaphore = require("saferphore"); var Util = require("../lib/common-util"); +var Meta = require("../lib/metadata"); +var Extras = require("../lib/hk-util"); const Schedule = require("../lib/schedule"); const Readline = require("readline"); @@ -39,6 +41,10 @@ var mkArchiveMetadataPath = function (env, channelId) { return Path.join(env.archiveRoot, 'datastore', channelId.slice(0, 2), channelId) + '.metadata.ndjson'; }; +var mkTempPath = function (env, channelId) { + return mkPath(env, channelId) + '.temp'; +}; + // pass in the path so we can reuse the same function for archived files var channelExists = function (filepath, cb) { Fs.stat(filepath, function (err, stat) { @@ -868,6 +874,178 @@ var getMessages = function (env, chanName, handler, cb) { }); }; +var trimChannel = function (env, channelName, hash, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + // this function is queued as a blocking action for the relevant channel + + // derive temporary file paths for metadata and log buffers + var tempChannelPath = mkTempPath(env, channelName); + + // derive production db paths + var channelPath = mkPath(env, channelName); + var metadataPath = mkMetadataPath(env, channelName); + + // derive archive paths + var archiveChannelPath = mkArchivePath(env, channelName); + var archiveMetadataPath = mkArchiveMetadataPath(env, channelName); + + var metadataReference = {}; + + var tempStream; + var ABORT; + + var cleanUp = function (cb) { + if (tempStream && !tempStream.closed) { + try { + tempStream.close(); + } catch (err) { } + } + + Fse.unlink(tempChannelPath, function (err) { + // proceed if deleted or if there was nothing to delete + if (!err || err.code === 'ENOENT') { return cb(); } + // else abort and call back with the error + cb(err); + }); + }; + + nThen(function (w) { + // close the file descriptor if it is open + closeChannel(env, channelName, w(function (err) { + if (err) { + w.abort(); + return void cb(err); + } + })); + }).nThen(function (w) { + cleanUp(w(function (err) { + if (err) { + w.abort(); + cb(err); + } + })); + }).nThen(function (w) { + // eat errors since loading the logger here would create a cyclical dependency + var lineHandler = Meta.createLineHandler(metadataReference, Util.noop); + + readMetadata(env, channelName, lineHandler, w(function (err) { + if (err) { + w.abort(); + return void cb(err); + } + // if there were no errors just fall through to the next block + })); + }).nThen(function (w) { + // create temp buffer writeStream + tempStream = Fs.createWriteStream(tempChannelPath, { + flags: 'a', + }); + tempStream.on('open', w()); + tempStream.on('error', function (err) { + w.abort(); + ABORT = true; + cleanUp(function () { + cb(err); + }); + }); + }).nThen(function (w) { + var i = 0; + var retain = false; + + var handler = function (msgObj, readMore, abort) { + if (ABORT) { return void abort(); } + // the first message might be metadata... ignore it if so + if (i++ === 0 && msgObj.buff.indexOf('{') === 0) { return; } + + if (retain) { + // if this flag is set then you've already found + // the message you were looking for. + // write it to your temp buffer and keep going + return void tempStream.write(msgObj.buff, function () { + readMore(); + }); + } + + var msg = Util.tryParse(msgObj.buff.toString('utf8')); + + var msgHash = Extras.getHash(msg[4]); + + if (msgHash === hash) { + // everything from this point on should be retained + retain = true; + } + }; + + readMessagesBin(env, channelName, 0, handler, w(function (err) { + if (err) { + w.abort(); + return void cleanUp(function () { + // intentionally call back with main error + // not the cleanup error + cb(err); + }); + } + + if (!retain) { + // you never found the message you were looking for + // this whole operation is invalid... + // clean up, abort, and call back with an error + + w.abort(); + cleanUp(function () { + // intentionally call back with main error + // not the cleanup error + cb('HASH_NOT_FOUND'); + }); + } + })); + }).nThen(function (w) { + // copy existing channel to the archive + Fse.copy(channelPath, archiveChannelPath, w(function (err) { + if (!err || err.code === 'ENOENT') { return; } + w.abort(); + cleanUp(function () { + cb(err); + }); + })); + + // copy existing metadaata to the archive + Fse.copy(metadataPath, archiveMetadataPath, w(function (err) { + if (!err || err.code === 'ENOENT') { return; } + w.abort(); + cleanUp(function () { + cb(err); + }); + })); + }).nThen(function (w) { + // overwrite the existing metadata log with the current metadata state + Fs.writeFile(metadataPath, JSON.stringify(metadataReference.meta) + '\n', w(function (err) { + // this shouldn't happen, but if it does your channel might be messed up :( + if (err) { + w.abort(); + cb(err); + } + })); + + // overwrite the existing channel with the temp log + Fse.move(tempChannelPath, channelPath, { + overwrite: true, + }, w(function (err) { + // this shouldn't happen, but if it does your channel might be messed up :( + if (err) { + w.abort(); + cb(err); + } + })); + }).nThen(function () { + // clean up and call back with no error + // triggering a historyKeeper index cache eviction... + cleanUp(function () { + cb(); + }); + }); +}; + module.exports.create = function (conf, cb) { var env = { root: conf.filePath || './datastore', @@ -987,25 +1165,9 @@ module.exports.create = function (conf, cb) { }); }, trimChannel: function (channelName, hash, cb) { - // XXX ansuz - // XXX queue lock - /* block any reads from the metadata and log files - until this whole process has finished - close the file descriptor if it is open - derive temporary file paths for metadata and log buffers - compute metadata state and write to metadata buffer - scan through log file and begin copying lines to the log buffer - once you recognize the first line by the hash the user provided - archive the file and current metadata once both buffers are copied - move the metadata and log buffers into place - return the lock on reads - call back - - in case of an error, remove the buffer files - */ + if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } schedule.blocking(channelName, function (next) { - cb("E_NOT_IMPLEMENTED"); - next(); + trimChannel(env, channelName, hash, Util.both(cb, next)); }); }, From 39b0785406a18a32c2a1a75c0c64ef036ca9dccd Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 09:57:12 -0500 Subject: [PATCH 020/172] apply custom limits immediately at startup --- lib/rpc.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/rpc.js b/lib/rpc.js index 0ece7915c..867cfd283 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -1754,6 +1754,7 @@ RPC.create = function (config, cb) { if (e) { WARN('limitUpdate', e); } + applyCustomLimits(Env, config); }); }; updateLimitDaily(); From 4a2b0fc114d5277aaa443105c861f6a6c0e4e640 Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 24 Jan 2020 15:58:42 +0100 Subject: [PATCH 021/172] Allow ooslide and oodoc imports --- www/common/onlyoffice/inner.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/www/common/onlyoffice/inner.js b/www/common/onlyoffice/inner.js index 53ff82307..bd08de807 100644 --- a/www/common/onlyoffice/inner.js +++ b/www/common/onlyoffice/inner.js @@ -1399,10 +1399,18 @@ define([ $exportXLSX.appendTo($rightside); var accept = [".bin", ".ods", ".xlsx"]; + if (type === "ooslide") { + accept = ['.bin', '.odp', '.pptx']; + } else if (type === "oodoc") { + accept = ['.bin', '.odt', '.docx']; + } if (typeof(Atomics) === "undefined") { accept = ['.bin']; } - var $importXLSX = common.createButton('import', true, { accept: accept, binary : ["ods", "xlsx"] }, importXLSXFile); + var $importXLSX = common.createButton('import', true, { + accept: accept, + binary : ["ods", "xlsx", "odt", "docx", "odp", "pptx"] + }, importXLSXFile); $importXLSX.appendTo($rightside); if (common.isLoggedIn()) { From 009bbd69bdbb52e8ca6d04767f0efc865a6d60e5 Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 24 Jan 2020 16:25:47 +0100 Subject: [PATCH 022/172] Fix import button in onlyoffice --- www/common/onlyoffice/inner.js | 1 + 1 file changed, 1 insertion(+) diff --git a/www/common/onlyoffice/inner.js b/www/common/onlyoffice/inner.js index bd08de807..71f43f3a1 100644 --- a/www/common/onlyoffice/inner.js +++ b/www/common/onlyoffice/inner.js @@ -1398,6 +1398,7 @@ define([ var $exportXLSX = common.createButton('export', true, {}, exportXLSXFile); $exportXLSX.appendTo($rightside); + var type = common.getMetadataMgr().getPrivateData().ooType; var accept = [".bin", ".ods", ".xlsx"]; if (type === "ooslide") { accept = ['.bin', '.odp', '.pptx']; From 1ecb61fe852d45b405c980be3ed3365f3c931dec Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 10:30:23 -0500 Subject: [PATCH 023/172] put an ugly red border on support thread messages from admins --- www/admin/app-admin.less | 3 +++ www/support/ui.js | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/www/admin/app-admin.less b/www/admin/app-admin.less index 10e178308..e9ba67325 100644 --- a/www/admin/app-admin.less +++ b/www/admin/app-admin.less @@ -23,5 +23,8 @@ display: flex; flex-flow: column; } + .cp-support-fromadmin { + border: 1px solid red !important; // XXX + } } diff --git a/www/support/ui.js b/www/support/ui.js index 42ee89c94..c008e219c 100644 --- a/www/support/ui.js +++ b/www/support/ui.js @@ -170,7 +170,9 @@ define([ var privateData = metadataMgr.getPrivateData(); // Check content.sender to see if it comes from us or from an admin - var fromMe = content.sender && content.sender.edPublic === privateData.edPublic; + var senderKey = content.sender && content.sender.edPublic; + var fromMe = senderKey === privateData.edPublic; + var fromAdmin = ctx.adminKeys.indexOf(senderKey) !== -1; var userData = h('div.cp-support-showdata', [ Messages.support_showData, @@ -183,7 +185,7 @@ define([ }); var name = Util.fixHTML(content.sender.name) || Messages.anonymous; - return h('div.cp-support-list-message', { + return h('div.cp-support-list-message' + (fromAdmin? '.cp-support-fromadmin': ''), { 'data-hash': hash }, [ h('div.cp-support-message-from' + (fromMe ? '.cp-support-fromme' : ''), [ @@ -219,6 +221,7 @@ define([ common: common, isAdmin: isAdmin, pinUsage: pinUsage || false, + adminKeys: Array.isArray(ApiConfig.adminKeys)? ApiConfig.adminKeys.slice(): [], }; ui.sendForm = function (id, form, dest) { From ba6e3f33bd25282d23849bab7853cf3d3d70f515 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 11:25:48 -0500 Subject: [PATCH 024/172] move admin commands into their own module --- lib/commands/admin-rpc.js | 121 ++++++++++++++++++++++++++++++++++++++ lib/rpc.js | 116 ++---------------------------------- 2 files changed, 125 insertions(+), 112 deletions(-) create mode 100644 lib/commands/admin-rpc.js diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js new file mode 100644 index 000000000..9e32fe2a8 --- /dev/null +++ b/lib/commands/admin-rpc.js @@ -0,0 +1,121 @@ +/*jshint esversion: 6 */ +const BatchRead = require("../batch-read"); +const nThen = require("nthen"); +const getFolderSize = require("get-folder-size"); +var Fs = require("fs"); + +var Admin = module.exports; + +var getActiveSessions = function (Env, ctx, cb) { + var total = ctx.users ? Object.keys(ctx.users).length : '?'; + + var ips = []; + Object.keys(ctx.users).forEach(function (u) { + var user = ctx.users[u]; + var socket = user.socket; + var req = socket.upgradeReq; + var conn = req && req.connection; + var ip = (req && req.headers && req.headers['x-forwarded-for']) || (conn && conn.remoteAddress); + if (ip && ips.indexOf(ip) === -1) { + ips.push(ip); + } + }); + + cb (void 0, [total, ips.length]); +}; + +var shutdown = function (Env, ctx, cb) { + return void cb('E_NOT_IMPLEMENTED'); + //clearInterval(Env.sessionExpirationInterval); + // XXX set a flag to prevent incoming database writes + // XXX disconnect all users and reject new connections + // XXX wait until all pending writes are complete + // then process.exit(0); + // and allow system functionality to restart the server +}; + +const batchRegisteredUsers = BatchRead("GET_REGISTERED_USERS"); +var getRegisteredUsers = function (Env, cb) { + batchRegisteredUsers('', cb, function (done) { + var dir = Env.paths.pin; + var folders; + var users = 0; + nThen(function (waitFor) { + Fs.readdir(dir, waitFor(function (err, list) { + if (err) { + waitFor.abort(); + return void done(err); + } + folders = list; + })); + }).nThen(function (waitFor) { + folders.forEach(function (f) { + var dir = Env.paths.pin + '/' + f; + Fs.readdir(dir, waitFor(function (err, list) { + if (err) { return; } + users += list.length; + })); + }); + }).nThen(function () { + done(void 0, users); + }); + }); +}; + +const batchDiskUsage = BatchRead("GET_DISK_USAGE"); +var getDiskUsage = function (Env, cb) { + batchDiskUsage('', cb, function (done) { + var data = {}; + nThen(function (waitFor) { + getFolderSize('./', waitFor(function(err, info) { + data.total = info; + })); + getFolderSize(Env.paths.pin, waitFor(function(err, info) { + data.pin = info; + })); + getFolderSize(Env.paths.blob, waitFor(function(err, info) { + data.blob = info; + })); + getFolderSize(Env.paths.staging, waitFor(function(err, info) { + data.blobstage = info; + })); + getFolderSize(Env.paths.block, waitFor(function(err, info) { + data.block = info; + })); + getFolderSize(Env.paths.data, waitFor(function(err, info) { + data.datastore = info; + })); + }).nThen(function () { + done(void 0, data); + }); + }); +}; + + + +Admin.command = function (Env, ctx, publicKey, config, data, cb) { + var admins = Env.admins; + if (admins.indexOf(publicKey) === -1) { + return void cb("FORBIDDEN"); + } + // Handle commands here + switch (data[0]) { + case 'ACTIVE_SESSIONS': + return getActiveSessions(Env, ctx, cb); + case 'ACTIVE_PADS': + return cb(void 0, ctx.channels ? Object.keys(ctx.channels).length : '?'); + case 'REGISTERED_USERS': + return getRegisteredUsers(Env, cb); + case 'DISK_USAGE': + return getDiskUsage(Env, cb); + case 'FLUSH_CACHE': + config.flushCache(); + return cb(void 0, true); + case 'SHUTDOWN': + return shutdown(Env, ctx, cb); + default: + return cb('UNHANDLED_ADMIN_COMMAND'); + } +}; + + diff --git a/lib/rpc.js b/lib/rpc.js index 867cfd283..f96e8a717 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -15,7 +15,6 @@ const Package = require('../package.json'); const Pinned = require('../scripts/pinned'); const Saferphore = require("saferphore"); const nThen = require("nthen"); -const getFolderSize = require("get-folder-size"); const Pins = require("./pins"); const Meta = require("./metadata"); const WriteQueue = require("./write-queue"); @@ -26,6 +25,8 @@ const escapeKeyCharacters = Util.escapeKeyCharacters; const unescapeKeyCharacters = Util.unescapeKeyCharacters; const mkEvent = Util.mkEvent; +const Admin = require("./commands/admin-rpc"); + var RPC = module.exports; var Store = require("../storage/file"); @@ -1207,115 +1208,6 @@ var writePrivateMessage = function (Env, args, nfwssCtx, cb) { }); }; -const batchDiskUsage = BatchRead("GET_DISK_USAGE"); -var getDiskUsage = function (Env, cb) { - batchDiskUsage('', cb, function (done) { - var data = {}; - nThen(function (waitFor) { - getFolderSize('./', waitFor(function(err, info) { - data.total = info; - })); - getFolderSize(Env.paths.pin, waitFor(function(err, info) { - data.pin = info; - })); - getFolderSize(Env.paths.blob, waitFor(function(err, info) { - data.blob = info; - })); - getFolderSize(Env.paths.staging, waitFor(function(err, info) { - data.blobstage = info; - })); - getFolderSize(Env.paths.block, waitFor(function(err, info) { - data.block = info; - })); - getFolderSize(Env.paths.data, waitFor(function(err, info) { - data.datastore = info; - })); - }).nThen(function () { - done(void 0, data); - }); - }); -}; - -const batchRegisteredUsers = BatchRead("GET_REGISTERED_USERS"); -var getRegisteredUsers = function (Env, cb) { - batchRegisteredUsers('', cb, function (done) { - var dir = Env.paths.pin; - var folders; - var users = 0; - nThen(function (waitFor) { - Fs.readdir(dir, waitFor(function (err, list) { - if (err) { - waitFor.abort(); - return void done(err); - } - folders = list; - })); - }).nThen(function (waitFor) { - folders.forEach(function (f) { - var dir = Env.paths.pin + '/' + f; - Fs.readdir(dir, waitFor(function (err, list) { - if (err) { return; } - users += list.length; - })); - }); - }).nThen(function () { - done(void 0, users); - }); - }); -}; -var getActiveSessions = function (Env, ctx, cb) { - var total = ctx.users ? Object.keys(ctx.users).length : '?'; - - var ips = []; - Object.keys(ctx.users).forEach(function (u) { - var user = ctx.users[u]; - var socket = user.socket; - var req = socket.upgradeReq; - var conn = req && req.connection; - var ip = (req && req.headers && req.headers['x-forwarded-for']) || (conn && conn.remoteAddress); - if (ip && ips.indexOf(ip) === -1) { - ips.push(ip); - } - }); - - cb (void 0, [total, ips.length]); -}; - -var shutdown = function (Env, ctx, cb) { - return void cb('E_NOT_IMPLEMENTED'); - //clearInterval(Env.sessionExpirationInterval); - // XXX set a flag to prevent incoming database writes - // XXX disconnect all users and reject new connections - // XXX wait until all pending writes are complete - // then process.exit(0); - // and allow system functionality to restart the server -}; - -var adminCommand = function (Env, ctx, publicKey, config, data, cb) { - var admins = Env.admins; - if (admins.indexOf(publicKey) === -1) { - return void cb("FORBIDDEN"); - } - // Handle commands here - switch (data[0]) { - case 'ACTIVE_SESSIONS': - return getActiveSessions(Env, ctx, cb); - case 'ACTIVE_PADS': - return cb(void 0, ctx.channels ? Object.keys(ctx.channels).length : '?'); - case 'REGISTERED_USERS': - return getRegisteredUsers(Env, cb); - case 'DISK_USAGE': - return getDiskUsage(Env, cb); - case 'FLUSH_CACHE': - config.flushCache(); - return cb(void 0, true); - case 'SHUTDOWN': - return shutdown(Env, ctx, cb); - default: - return cb('UNHANDLED_ADMIN_COMMAND'); - } -}; - var isUnauthenticatedCall = function (call) { return [ 'GET_FILE_SIZE', @@ -1717,7 +1609,7 @@ RPC.create = function (config, cb) { Respond(e); }); case 'ADMIN': - return void adminCommand(Env, ctx, safeKey, config, msg[1], function (e, result) { + return void Admin.command(Env, ctx, safeKey, config, msg[1], function (e, result) { if (e) { WARN(e, result); return void Respond(e); @@ -1754,9 +1646,9 @@ RPC.create = function (config, cb) { if (e) { WARN('limitUpdate', e); } - applyCustomLimits(Env, config); }); }; + applyCustomLimits(Env, config); updateLimitDaily(); setInterval(updateLimitDaily, 24*3600*1000); From 9a53b3b9fd69d03e1f1f2341bcc9b8dec0f3b171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Benqu=C3=A9?= Date: Fri, 24 Jan 2020 16:52:44 +0000 Subject: [PATCH 025/172] style messages - blue text for messages from admins - red text and background for messages needing a response. --- www/admin/app-admin.less | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/www/admin/app-admin.less b/www/admin/app-admin.less index e9ba67325..eb2a8c0f8 100644 --- a/www/admin/app-admin.less +++ b/www/admin/app-admin.less @@ -23,8 +23,26 @@ display: flex; flex-flow: column; } + + .cp-support-list-actions { + margin: 10px 0px 10px 2px; + } + .cp-support-list-message { + &:last-child:not(.cp-support-fromadmin) { + color: @colortheme_cp-red; + background-color: lighten(@colortheme_cp-red, 25%); + .cp-support-showdata { + background-color: lighten(@colortheme_cp-red, 30%); + } + } + } + .cp-support-fromadmin { - border: 1px solid red !important; // XXX + color: @colortheme_logo-2; + background-color: #FFF; + .cp-support-message-content { + color: @colortheme_logo-2; + } } } From 6b5118cdc3a2dff5f2637ef8da36f481439bdafe Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 12:41:42 -0500 Subject: [PATCH 026/172] add an npm script to lint only server components --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index fa4353662..057c823d9 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "package": "PACKAGE=1 node server.js", "lint": "jshint --config .jshintrc --exclude-path .jshintignore . && ./node_modules/lesshint/bin/lesshint -c ./.lesshintrc ./customize.dist/src/less2/", "lint:js": "jshint --config .jshintrc --exclude-path .jshintignore .", + "lint:server": "jshint --config .jshintrc lib", "lint:less": "./node_modules/lesshint/bin/lesshint -c ./.lesshintrc ./customize.dist/src/less2/", "flow": "./node_modules/.bin/flow", "test": "node scripts/TestSelenium.js", From 4fd68b672ec340019427aec6d021f49b4054e3cb Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 12:42:31 -0500 Subject: [PATCH 027/172] drop clientside hooks wrappers for authenticated GET_FILE_SIZE --- www/common/pinpad.js | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/www/common/pinpad.js b/www/common/pinpad.js index 48c62f3da..e9ecbb7f3 100644 --- a/www/common/pinpad.js +++ b/www/common/pinpad.js @@ -89,18 +89,6 @@ var factory = function (Util, Rpc) { }); }; - // get the total stored size of a channel's patches (in bytes) - exp.getFileSize = function (file, cb) { - rpc.send('GET_FILE_SIZE', file, function (e, response) { - if (e) { return void cb(e); } - if (response && response.length && typeof(response[0]) === 'number') { - return void cb(void 0, response[0]); - } else { - cb('INVALID_RESPONSE'); - } - }); - }; - // get the combined size of all channels (in bytes) for all the // channels which the server has pinned for your publicKey exp.getFileListSize = function (cb) { From c93b39c094e8e827024c4f655d144aa103d13be8 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 12:43:11 -0500 Subject: [PATCH 028/172] separate more rpc functionality into pinning and core submodules --- lib/commands/core.js | 42 ++++ lib/commands/pin-rpc.js | 399 +++++++++++++++++++++++++++++++++ lib/rpc.js | 479 +++------------------------------------- 3 files changed, 471 insertions(+), 449 deletions(-) create mode 100644 lib/commands/core.js create mode 100644 lib/commands/pin-rpc.js diff --git a/lib/commands/core.js b/lib/commands/core.js new file mode 100644 index 000000000..7e34232cb --- /dev/null +++ b/lib/commands/core.js @@ -0,0 +1,42 @@ +/*jshint esversion: 6 */ +const Core = module.exports; +const Util = require("../common-util"); +const escapeKeyCharacters = Util.escapeKeyCharacters; + +Core.DEFAULT_LIMIT = 50 * 1024 * 1024; +Core.SESSION_EXPIRATION_TIME = 60 * 1000; + +Core.isValidId = function (chan) { + return chan && chan.length && /^[a-zA-Z0-9=+-]*$/.test(chan) && + [32, 48].indexOf(chan.length) > -1; +}; + +var makeToken = Core.makeToken = function () { + return Number(Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)) + .toString(16); +}; + +Core.getSession = function (Sessions, key) { + var safeKey = escapeKeyCharacters(key); + if (Sessions[safeKey]) { + Sessions[safeKey].atime = +new Date(); + return Sessions[safeKey]; + } + var user = Sessions[safeKey] = {}; + user.atime = +new Date(); + user.tokens = [ + makeToken() + ]; + return user; +}; + + + +// getChannelList +// getSession +// getHash +// getMultipleFileSize +// sumChannelSizes +// getFreeSpace +// getLimit + diff --git a/lib/commands/pin-rpc.js b/lib/commands/pin-rpc.js new file mode 100644 index 000000000..3412f329e --- /dev/null +++ b/lib/commands/pin-rpc.js @@ -0,0 +1,399 @@ +/*jshint esversion: 6 */ +const Core = require("./core"); + +const BatchRead = require("../batch-read"); +const Pins = require("../pins"); + +const Pinning = module.exports; +const Nacl = require("tweetnacl/nacl-fast"); +const Util = require("../common-util"); +const nThen = require("nthen"); + +//const escapeKeyCharacters = Util.escapeKeyCharacters; +const unescapeKeyCharacters = Util.unescapeKeyCharacters; + +var sumChannelSizes = function (sizes) { + return Object.keys(sizes).map(function (id) { return sizes[id]; }) + .filter(function (x) { + // only allow positive numbers + return !(typeof(x) !== 'number' || x <= 0); + }) + .reduce(function (a, b) { return a + b; }, 0); +}; + +// XXX it's possible for this to respond before the server has had a chance +// to fetch the limits. Maybe we should respond with an error... +// or wait until we actually know the limits before responding +var getLimit = Pinning.getLimit = function (Env, publicKey, cb) { + var unescapedKey = unescapeKeyCharacters(publicKey); + var limit = Env.limits[unescapedKey]; + var defaultLimit = typeof(Env.defaultStorageLimit) === 'number'? + Env.defaultStorageLimit: Core.DEFAULT_LIMIT; + + var toSend = limit && typeof(limit.limit) === "number"? + [limit.limit, limit.plan, limit.note] : [defaultLimit, '', '']; + + cb(void 0, toSend); +}; + +var addPinned = function ( + Env, + publicKey /*:string*/, + channelList /*Array*/, + cb /*:()=>void*/) +{ + Env.evPinnedPadsReady.reg(() => { + channelList.forEach((c) => { + const x = Env.pinnedPads[c] = Env.pinnedPads[c] || {}; + x[publicKey] = 1; + }); + cb(); + }); +}; +var removePinned = function ( + Env, + publicKey /*:string*/, + channelList /*Array*/, + cb /*:()=>void*/) +{ + Env.evPinnedPadsReady.reg(() => { + channelList.forEach((c) => { + const x = Env.pinnedPads[c]; + if (!x) { return; } + delete x[publicKey]; + }); + cb(); + }); +}; + +var getMultipleFileSize = function (Env, channels, cb) { + if (!Array.isArray(channels)) { return cb('INVALID_PIN_LIST'); } + if (typeof(Env.msgStore.getChannelSize) !== 'function') { + return cb('GET_CHANNEL_SIZE_UNSUPPORTED'); + } + + var i = channels.length; + var counts = {}; + + var done = function () { + i--; + if (i === 0) { return cb(void 0, counts); } + }; + + channels.forEach(function (channel) { + Pinning.getFileSize(Env, channel, function (e, size) { + if (e) { + // most likely error here is that a file no longer exists + // but a user still has it in their drive, and wants to know + // its size. We should find a way to inform them of this in + // the future. For now we can just tell them it has no size. + + //WARN('getFileSize', e); + counts[channel] = 0; + return done(); + } + counts[channel] = size; + done(); + }); + }); +}; + +const batchUserPins = BatchRead("LOAD_USER_PINS"); +var loadUserPins = function (Env, publicKey, cb) { + var session = Core.getSession(Env.Sessions, publicKey); + + if (session.channels) { + return cb(session.channels); + } + + batchUserPins(publicKey, cb, function (done) { + var ref = {}; + var lineHandler = Pins.createLineHandler(ref, function (label, data) { + Env.Log.error(label, { + log: publicKey, + data: data, + }); + }); + + // if channels aren't in memory. load them from disk + Env.pinStore.getMessages(publicKey, lineHandler, function () { + // no more messages + + // only put this into the cache if it completes + session.channels = ref.pins; + done(ref.pins); // FIXME no error handling? + }); + }); +}; + +var truthyKeys = function (O) { + return Object.keys(O).filter(function (k) { + return O[k]; + }); +}; + +var getChannelList = Pinning.getChannelList = function (Env, publicKey, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + loadUserPins(Env, publicKey, function (pins) { + cb(truthyKeys(pins)); + }); +}; + +const batchTotalSize = BatchRead("GET_TOTAL_SIZE"); +Pinning.getTotalSize = function (Env, publicKey, cb) { + var unescapedKey = unescapeKeyCharacters(publicKey); + var limit = Env.limits[unescapedKey]; + + // Get a common key if multiple users share the same quota, otherwise take the public key + var batchKey = (limit && Array.isArray(limit.users)) ? limit.users.join('') : publicKey; + + batchTotalSize(batchKey, cb, function (done) { + var channels = []; + var bytes = 0; + nThen(function (waitFor) { + // Get the channels list for our user account + Pinning.getChannelList(Env, publicKey, waitFor(function (_channels) { + if (!_channels) { + waitFor.abort(); + return done('INVALID_PIN_LIST'); + } + Array.prototype.push.apply(channels, _channels); + })); + // Get the channels list for users sharing our quota + if (limit && Array.isArray(limit.users) && limit.users.length > 1) { + limit.users.forEach(function (key) { + if (key === unescapedKey) { return; } // Don't count ourselves twice + getChannelList(Env, key, waitFor(function (_channels) { + if (!_channels) { return; } // Broken user, don't count their quota + Array.prototype.push.apply(channels, _channels); + })); + }); + } + }).nThen(function (waitFor) { + // Get size of the channels + var list = []; // Contains the channels already counted in the quota to avoid duplicates + channels.forEach(function (channel) { // TODO semaphore? + if (list.indexOf(channel) !== -1) { return; } + list.push(channel); + Pinning.getFileSize(Env, channel, waitFor(function (e, size) { + if (!e) { bytes += size; } + })); + }); + }).nThen(function () { + done(void 0, bytes); + }); + }); +}; + +/* Users should be able to clear their own pin log with an authenticated RPC +*/ +Pinning.removePins = function (Env, safeKey, cb) { + if (typeof(Env.pinStore.removeChannel) !== 'function') { + return void cb("E_NOT_IMPLEMENTED"); + } + Env.pinStore.removeChannel(safeKey, function (err) { + Env.Log.info('DELETION_PIN_BY_OWNER_RPC', { + safeKey: safeKey, + status: err? String(err): 'SUCCESS', + }); + + cb(err); + }); +}; + +Pinning.trimPins = function (Env, safeKey, cb) { + // XXX trim to latest pin checkpoint + cb("NOT_IMPLEMENTED"); +}; + +var getFreeSpace = Pinning.getFreeSpace = function (Env, publicKey, cb) { + getLimit(Env, publicKey, function (e, limit) { + if (e) { return void cb(e); } + Pinning.getTotalSize(Env, publicKey, function (e, size) { + if (typeof(size) === 'undefined') { return void cb(e); } + + var rem = limit[0] - size; + if (typeof(rem) !== 'number') { + return void cb('invalid_response'); + } + cb(void 0, rem); + }); + }); +}; + +var hashChannelList = function (A) { + var uniques = []; + + A.forEach(function (a) { + if (uniques.indexOf(a) === -1) { uniques.push(a); } + }); + uniques.sort(); + + var hash = Nacl.util.encodeBase64(Nacl.hash(Nacl + .util.decodeUTF8(JSON.stringify(uniques)))); + + return hash; +}; + +var getHash = Pinning.getHash = function (Env, publicKey, cb) { + getChannelList(Env, publicKey, function (channels) { + cb(void 0, hashChannelList(channels)); + }); +}; + +Pinning.pinChannel = function (Env, publicKey, channels, cb) { + if (!channels && channels.filter) { + return void cb('INVALID_PIN_LIST'); + } + + // get channel list ensures your session has a cached channel list + getChannelList(Env, publicKey, function (pinned) { + var session = Core.getSession(Env.Sessions, publicKey); + + // only pin channels which are not already pinned + var toStore = channels.filter(function (channel) { + return pinned.indexOf(channel) === -1; + }); + + if (toStore.length === 0) { + return void getHash(Env, publicKey, cb); + } + + getMultipleFileSize(Env, toStore, function (e, sizes) { + if (typeof(sizes) === 'undefined') { return void cb(e); } + var pinSize = sumChannelSizes(sizes); + + getFreeSpace(Env, publicKey, function (e, free) { + if (typeof(free) === 'undefined') { + Env.WARN('getFreeSpace', e); + return void cb(e); + } + if (pinSize > free) { return void cb('E_OVER_LIMIT'); } + + Env.pinStore.message(publicKey, JSON.stringify(['PIN', toStore, +new Date()]), + function (e) { + if (e) { return void cb(e); } + toStore.forEach(function (channel) { + session.channels[channel] = true; + }); + addPinned(Env, publicKey, toStore, () => {}); + getHash(Env, publicKey, cb); + }); + }); + }); + }); +}; + +Pinning.unpinChannel = function (Env, publicKey, channels, cb) { + if (!channels && channels.filter) { + // expected array + return void cb('INVALID_PIN_LIST'); + } + + getChannelList(Env, publicKey, function (pinned) { + var session = Core.getSession(Env.Sessions, publicKey); + + // only unpin channels which are pinned + var toStore = channels.filter(function (channel) { + return pinned.indexOf(channel) !== -1; + }); + + if (toStore.length === 0) { + return void getHash(Env, publicKey, cb); + } + + Env.pinStore.message(publicKey, JSON.stringify(['UNPIN', toStore, +new Date()]), + function (e) { + if (e) { return void cb(e); } + toStore.forEach(function (channel) { + delete session.channels[channel]; + }); + removePinned(Env, publicKey, toStore, () => {}); + getHash(Env, publicKey, cb); + }); + }); +}; + +Pinning.resetUserPins = function (Env, publicKey, channelList, cb) { + if (!Array.isArray(channelList)) { return void cb('INVALID_PIN_LIST'); } + var session = Core.getSession(Env.Sessions, publicKey); + + if (!channelList.length) { + return void getHash(Env, publicKey, function (e, hash) { + if (e) { return cb(e); } + cb(void 0, hash); + }); + } + + var pins = {}; + getMultipleFileSize(Env, channelList, function (e, sizes) { + if (typeof(sizes) === 'undefined') { return void cb(e); } + var pinSize = sumChannelSizes(sizes); + + + getLimit(Env, publicKey, function (e, limit) { + if (e) { + Env.WARN('[RESET_ERR]', e); + return void cb(e); + } + + /* we want to let people pin, even if they are over their limit, + but they should only be able to do this once. + + This prevents data loss in the case that someone registers, but + does not have enough free space to pin their migrated data. + + They will not be able to pin additional pads until they upgrade + or delete enough files to go back under their limit. */ + if (pinSize > limit[0] && session.hasPinned) { return void(cb('E_OVER_LIMIT')); } + Env.pinStore.message(publicKey, JSON.stringify(['RESET', channelList, +new Date()]), + function (e) { + if (e) { return void cb(e); } + channelList.forEach(function (channel) { + pins[channel] = true; + }); + + var oldChannels; + if (session.channels && typeof(session.channels) === 'object') { + oldChannels = Object.keys(session.channels); + } else { + oldChannels = []; + } + removePinned(Env, publicKey, oldChannels, () => { + addPinned(Env, publicKey, channelList, ()=>{}); + }); + + // update in-memory cache IFF the reset was allowed. + session.channels = pins; + getHash(Env, publicKey, function (e, hash) { + cb(e, hash); + }); + }); + }); + }); +}; + +Pinning.getFileSize = function (Env, channel, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } + if (channel.length === 32) { + if (typeof(Env.msgStore.getChannelSize) !== 'function') { + return cb('GET_CHANNEL_SIZE_UNSUPPORTED'); + } + + return void Env.msgStore.getChannelSize(channel, function (e, size /*:number*/) { + if (e) { + if (e.code === 'ENOENT') { return void cb(void 0, 0); } + return void cb(e.code); + } + cb(void 0, size); + }); + } + + // 'channel' refers to a file, so you need another API + Env.blobStore.size(channel, function (e, size) { + if (typeof(size) === 'undefined') { return void cb(e); } + cb(void 0, size); + }); +}; + diff --git a/lib/rpc.js b/lib/rpc.js index f96e8a717..a064ebe64 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -15,10 +15,10 @@ const Package = require('../package.json'); const Pinned = require('../scripts/pinned'); const Saferphore = require("saferphore"); const nThen = require("nthen"); -const Pins = require("./pins"); const Meta = require("./metadata"); const WriteQueue = require("./write-queue"); const BatchRead = require("./batch-read"); +const Core = require("./commands/core"); const Util = require("./common-util"); const escapeKeyCharacters = Util.escapeKeyCharacters; @@ -26,15 +26,13 @@ const unescapeKeyCharacters = Util.unescapeKeyCharacters; const mkEvent = Util.mkEvent; const Admin = require("./commands/admin-rpc"); +const Pinning = require("./commands/pin-rpc"); var RPC = module.exports; var Store = require("../storage/file"); var BlobStore = require("../storage/blob"); -var DEFAULT_LIMIT = 50 * 1024 * 1024; -var SESSION_EXPIRATION_TIME = 60 * 1000; - var Log; var WARN = function (e, output) { @@ -47,16 +45,6 @@ var WARN = function (e, output) { } }; -var isValidId = function (chan) { - return chan && chan.length && /^[a-zA-Z0-9=+-]*$/.test(chan) && - [32, 48].indexOf(chan.length) > -1; -}; - -var makeToken = function () { - return Number(Math.floor(Math.random() * Number.MAX_SAFE_INTEGER)) - .toString(16); -}; - var makeCookie = function (token) { var time = (+new Date()); time -= time % 5000; @@ -81,20 +69,6 @@ var parseCookie = function (cookie) { return c; }; -var getSession = function (Sessions, key) { - var safeKey = escapeKeyCharacters(key); - if (Sessions[safeKey]) { - Sessions[safeKey].atime = +new Date(); - return Sessions[safeKey]; - } - var user = Sessions[safeKey] = {}; - user.atime = +new Date(); - user.tokens = [ - makeToken() - ]; - return user; -}; - var isTooOld = function (time, now) { return (now - time) > 300000; }; @@ -121,7 +95,7 @@ var expireSessions = function (Sessions) { var addTokenForKey = function (Sessions, publicKey, token) { if (!Sessions[publicKey]) { throw new Error('undefined user'); } - var user = getSession(Sessions, publicKey); + var user = Core.getSession(Sessions, publicKey); user.tokens.push(token); user.atime = +new Date(); if (user.tokens.length > 2) { user.tokens.shift(); } @@ -143,7 +117,7 @@ var isValidCookie = function (Sessions, publicKey, cookie) { return false; } - var user = getSession(Sessions, publicKey); + var user = Core.getSession(Sessions, publicKey); if (!user) { return false; } var idx = user.tokens.indexOf(parsed.seq); @@ -151,7 +125,7 @@ var isValidCookie = function (Sessions, publicKey, cookie) { if (idx > 0) { // make a new token - addTokenForKey(Sessions, publicKey, makeToken()); + addTokenForKey(Sessions, publicKey, Core.makeToken()); } return true; @@ -195,74 +169,9 @@ var checkSignature = function (signedMsg, signature, publicKey) { return Nacl.sign.detached.verify(signedBuffer, signatureBuffer, pubBuffer); }; -const batchUserPins = BatchRead("LOAD_USER_PINS"); -var loadUserPins = function (Env, publicKey, cb) { - var session = getSession(Env.Sessions, publicKey); - - if (session.channels) { - return cb(session.channels); - } - - batchUserPins(publicKey, cb, function (done) { - var ref = {}; - var lineHandler = Pins.createLineHandler(ref, function (label, data) { - Log.error(label, { - log: publicKey, - data: data, - }); - }); - - // if channels aren't in memory. load them from disk - Env.pinStore.getMessages(publicKey, lineHandler, function () { - // no more messages - - // only put this into the cache if it completes - session.channels = ref.pins; - done(ref.pins); // FIXME no error handling? - }); - }); -}; - -var truthyKeys = function (O) { - return Object.keys(O).filter(function (k) { - return O[k]; - }); -}; - -var getChannelList = function (Env, publicKey, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - loadUserPins(Env, publicKey, function (pins) { - cb(truthyKeys(pins)); - }); -}; - -var getFileSize = function (Env, channel, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - if (!isValidId(channel)) { return void cb('INVALID_CHAN'); } - if (channel.length === 32) { - if (typeof(Env.msgStore.getChannelSize) !== 'function') { - return cb('GET_CHANNEL_SIZE_UNSUPPORTED'); - } - - return void Env.msgStore.getChannelSize(channel, function (e, size /*:number*/) { - if (e) { - if (e.code === 'ENOENT') { return void cb(void 0, 0); } - return void cb(e.code); - } - cb(void 0, size); - }); - } - - // 'channel' refers to a file, so you need another API - Env.blobStore.size(channel, function (e, size) { - if (typeof(size) === 'undefined') { return void cb(e); } - cb(void 0, size); - }); -}; - const batchMetadata = BatchRead("GET_METADATA"); var getMetadata = function (Env, channel, cb) { - if (!isValidId(channel)) { return void cb('INVALID_CHAN'); } + if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } if (channel.length !== 32) { return cb("INVALID_CHAN_LENGTH"); } batchMetadata(channel, cb, function (done) { @@ -309,7 +218,7 @@ var queueMetadata = WriteQueue(); var setMetadata = function (Env, data, unsafeKey, cb) { var channel = data.channel; var command = data.command; - if (!channel || !isValidId(channel)) { return void cb ('INVALID_CHAN'); } + if (!channel || !Core.isValidId(channel)) { return void cb ('INVALID_CHAN'); } if (!command || typeof (command) !== 'string') { return void cb ('INVALID_COMMAND'); } if (Meta.commands.indexOf(command) === -1) { return void('UNSUPPORTED_COMMAND'); } @@ -382,38 +291,6 @@ var setMetadata = function (Env, data, unsafeKey, cb) { }); }; -var getMultipleFileSize = function (Env, channels, cb) { - if (!Array.isArray(channels)) { return cb('INVALID_PIN_LIST'); } - if (typeof(Env.msgStore.getChannelSize) !== 'function') { - return cb('GET_CHANNEL_SIZE_UNSUPPORTED'); - } - - var i = channels.length; - var counts = {}; - - var done = function () { - i--; - if (i === 0) { return cb(void 0, counts); } - }; - - channels.forEach(function (channel) { - getFileSize(Env, channel, function (e, size) { - if (e) { - // most likely error here is that a file no longer exists - // but a user still has it in their drive, and wants to know - // its size. We should find a way to inform them of this in - // the future. For now we can just tell them it has no size. - - //WARN('getFileSize', e); - counts[channel] = 0; - return done(); - } - counts[channel] = size; - done(); - }); - }); -}; - /* accepts a list, and returns a sublist of channel or file ids which seem to have been deleted from the server (file size 0) @@ -429,7 +306,7 @@ var getDeletedPads = function (Env, channels, cb) { var job = function (channel, wait) { return function (give) { - getFileSize(Env, channel, wait(give(function (e, size) { + Pinning.getFileSize(Env, channel, wait(give(function (e, size) { if (e) { return; } if (size === 0) { absentees.push(channel); } }))); @@ -445,72 +322,6 @@ var getDeletedPads = function (Env, channels, cb) { }); }; -const batchTotalSize = BatchRead("GET_TOTAL_SIZE"); -var getTotalSize = function (Env, publicKey, cb) { - var unescapedKey = unescapeKeyCharacters(publicKey); - var limit = Env.limits[unescapedKey]; - - // Get a common key if multiple users share the same quota, otherwise take the public key - var batchKey = (limit && Array.isArray(limit.users)) ? limit.users.join('') : publicKey; - - batchTotalSize(batchKey, cb, function (done) { - var channels = []; - var bytes = 0; - nThen(function (waitFor) { - // Get the channels list for our user account - getChannelList(Env, publicKey, waitFor(function (_channels) { - if (!_channels) { - waitFor.abort(); - return done('INVALID_PIN_LIST'); - } - Array.prototype.push.apply(channels, _channels); - })); - // Get the channels list for users sharing our quota - if (limit && Array.isArray(limit.users) && limit.users.length > 1) { - limit.users.forEach(function (key) { - if (key === unescapedKey) { return; } // Don't count ourselves twice - getChannelList(Env, key, waitFor(function (_channels) { - if (!_channels) { return; } // Broken user, don't count their quota - Array.prototype.push.apply(channels, _channels); - })); - }); - } - }).nThen(function (waitFor) { - // Get size of the channels - var list = []; // Contains the channels already counted in the quota to avoid duplicates - channels.forEach(function (channel) { // TODO semaphore? - if (list.indexOf(channel) !== -1) { return; } - list.push(channel); - getFileSize(Env, channel, waitFor(function (e, size) { - if (!e) { bytes += size; } - })); - }); - }).nThen(function () { - done(void 0, bytes); - }); - }); -}; - -var hashChannelList = function (A) { - var uniques = []; - - A.forEach(function (a) { - if (uniques.indexOf(a) === -1) { uniques.push(a); } - }); - uniques.sort(); - - var hash = Nacl.util.encodeBase64(Nacl.hash(Nacl - .util.decodeUTF8(JSON.stringify(uniques)))); - - return hash; -}; - -var getHash = function (Env, publicKey, cb) { - getChannelList(Env, publicKey, function (channels) { - cb(void 0, hashChannelList(channels)); - }); -}; - var applyCustomLimits = function (Env, config) { var isLimit = function (o) { var valid = o && typeof(o) === 'object' && @@ -551,7 +362,7 @@ var updateLimits = function (Env, config, publicKey, cb /*:(?string, ?any[])=>vo if (typeof cb !== "function") { cb = function () {}; } var defaultLimit = typeof(config.defaultStorageLimit) === 'number'? - config.defaultStorageLimit: DEFAULT_LIMIT; + config.defaultStorageLimit: Core.DEFAULT_LIMIT; var userId; if (publicKey) { @@ -612,45 +423,6 @@ var updateLimits = function (Env, config, publicKey, cb /*:(?string, ?any[])=>vo req.end(body); }; -// XXX it's possible for this to respond before the server has had a chance -// to fetch the limits. Maybe we should respond with an error... -// or wait until we actually know the limits before responding -var getLimit = function (Env, publicKey, cb) { - var unescapedKey = unescapeKeyCharacters(publicKey); - var limit = Env.limits[unescapedKey]; - var defaultLimit = typeof(Env.defaultStorageLimit) === 'number'? - Env.defaultStorageLimit: DEFAULT_LIMIT; - - var toSend = limit && typeof(limit.limit) === "number"? - [limit.limit, limit.plan, limit.note] : [defaultLimit, '', '']; - - cb(void 0, toSend); -}; - -var getFreeSpace = function (Env, publicKey, cb) { - getLimit(Env, publicKey, function (e, limit) { - if (e) { return void cb(e); } - getTotalSize(Env, publicKey, function (e, size) { - if (typeof(size) === 'undefined') { return void cb(e); } - - var rem = limit[0] - size; - if (typeof(rem) !== 'number') { - return void cb('invalid_response'); - } - cb(void 0, rem); - }); - }); -}; - -var sumChannelSizes = function (sizes) { - return Object.keys(sizes).map(function (id) { return sizes[id]; }) - .filter(function (x) { - // only allow positive numbers - return !(typeof(x) !== 'number' || x <= 0); - }) - .reduce(function (a, b) { return a + b; }, 0); -}; - // inform that the var loadChannelPins = function (Env) { Pinned.load(function (err, data) { @@ -670,35 +442,7 @@ var loadChannelPins = function (Env) { pinPath: Env.paths.pin, }); }; -var addPinned = function ( - Env, - publicKey /*:string*/, - channelList /*Array*/, - cb /*:()=>void*/) -{ - Env.evPinnedPadsReady.reg(() => { - channelList.forEach((c) => { - const x = Env.pinnedPads[c] = Env.pinnedPads[c] || {}; - x[publicKey] = 1; - }); - cb(); - }); -}; -var removePinned = function ( - Env, - publicKey /*:string*/, - channelList /*Array*/, - cb /*:()=>void*/) -{ - Env.evPinnedPadsReady.reg(() => { - channelList.forEach((c) => { - const x = Env.pinnedPads[c]; - if (!x) { return; } - delete x[publicKey]; - }); - cb(); - }); -}; + var isChannelPinned = function (Env, channel, cb) { Env.evPinnedPadsReady.reg(() => { if (Env.pinnedPads[channel] && Object.keys(Env.pinnedPads[channel]).length) { @@ -710,138 +454,6 @@ var isChannelPinned = function (Env, channel, cb) { }); }; -var pinChannel = function (Env, publicKey, channels, cb) { - if (!channels && channels.filter) { - return void cb('INVALID_PIN_LIST'); - } - - // get channel list ensures your session has a cached channel list - getChannelList(Env, publicKey, function (pinned) { - var session = getSession(Env.Sessions, publicKey); - - // only pin channels which are not already pinned - var toStore = channels.filter(function (channel) { - return pinned.indexOf(channel) === -1; - }); - - if (toStore.length === 0) { - return void getHash(Env, publicKey, cb); - } - - getMultipleFileSize(Env, toStore, function (e, sizes) { - if (typeof(sizes) === 'undefined') { return void cb(e); } - var pinSize = sumChannelSizes(sizes); - - getFreeSpace(Env, publicKey, function (e, free) { - if (typeof(free) === 'undefined') { - WARN('getFreeSpace', e); - return void cb(e); - } - if (pinSize > free) { return void cb('E_OVER_LIMIT'); } - - Env.pinStore.message(publicKey, JSON.stringify(['PIN', toStore, +new Date()]), - function (e) { - if (e) { return void cb(e); } - toStore.forEach(function (channel) { - session.channels[channel] = true; - }); - addPinned(Env, publicKey, toStore, () => {}); - getHash(Env, publicKey, cb); - }); - }); - }); - }); -}; - -var unpinChannel = function (Env, publicKey, channels, cb) { - if (!channels && channels.filter) { - // expected array - return void cb('INVALID_PIN_LIST'); - } - - getChannelList(Env, publicKey, function (pinned) { - var session = getSession(Env.Sessions, publicKey); - - // only unpin channels which are pinned - var toStore = channels.filter(function (channel) { - return pinned.indexOf(channel) !== -1; - }); - - if (toStore.length === 0) { - return void getHash(Env, publicKey, cb); - } - - Env.pinStore.message(publicKey, JSON.stringify(['UNPIN', toStore, +new Date()]), - function (e) { - if (e) { return void cb(e); } - toStore.forEach(function (channel) { - delete session.channels[channel]; - }); - removePinned(Env, publicKey, toStore, () => {}); - getHash(Env, publicKey, cb); - }); - }); -}; - -var resetUserPins = function (Env, publicKey, channelList, cb) { - if (!Array.isArray(channelList)) { return void cb('INVALID_PIN_LIST'); } - var session = getSession(Env.Sessions, publicKey); - - if (!channelList.length) { - return void getHash(Env, publicKey, function (e, hash) { - if (e) { return cb(e); } - cb(void 0, hash); - }); - } - - var pins = {}; - getMultipleFileSize(Env, channelList, function (e, sizes) { - if (typeof(sizes) === 'undefined') { return void cb(e); } - var pinSize = sumChannelSizes(sizes); - - - getLimit(Env, publicKey, function (e, limit) { - if (e) { - WARN('[RESET_ERR]', e); - return void cb(e); - } - - /* we want to let people pin, even if they are over their limit, - but they should only be able to do this once. - - This prevents data loss in the case that someone registers, but - does not have enough free space to pin their migrated data. - - They will not be able to pin additional pads until they upgrade - or delete enough files to go back under their limit. */ - if (pinSize > limit[0] && session.hasPinned) { return void(cb('E_OVER_LIMIT')); } - Env.pinStore.message(publicKey, JSON.stringify(['RESET', channelList, +new Date()]), - function (e) { - if (e) { return void cb(e); } - channelList.forEach(function (channel) { - pins[channel] = true; - }); - - var oldChannels; - if (session.channels && typeof(session.channels) === 'object') { - oldChannels = Object.keys(session.channels); - } else { - oldChannels = []; - } - removePinned(Env, publicKey, oldChannels, () => { - addPinned(Env, publicKey, channelList, ()=>{}); - }); - - // update in-memory cache IFF the reset was allowed. - session.channels = pins; - getHash(Env, publicKey, function (e, hash) { - cb(e, hash); - }); - }); - }); - }); -}; - var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { if (typeof(channelId) !== 'string' || channelId.length !== 32) { return cb('INVALID_ARGUMENTS'); @@ -861,7 +473,7 @@ var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { }; var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { - if (typeof(channelId) !== 'string' || !isValidId(channelId)) { + if (typeof(channelId) !== 'string' || !Core.isValidId(channelId)) { return cb('INVALID_ARGUMENTS'); } @@ -948,26 +560,6 @@ var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { }); }; -/* Users should be able to clear their own pin log with an authenticated RPC -*/ -var removePins = function (Env, safeKey, cb) { - if (typeof(Env.pinStore.removeChannel) !== 'function') { - return void cb("E_NOT_IMPLEMENTED"); - } - Env.pinStore.removeChannel(safeKey, function (err) { - Log.info('DELETION_PIN_BY_OWNER_RPC', { - safeKey: safeKey, - status: err? String(err): 'SUCCESS', - }); - - cb(err); - }); -}; - -var trimPins = function (Env, safeKey, cb) { - // XXX trim to latest pin checkpoint - cb("NOT_IMPLEMENTED"); -}; /* We assume that the server is secured against MitM attacks @@ -1136,7 +728,7 @@ var ARRAY_LINE = /^\[/; otherwise false */ var isNewChannel = function (Env, channel, cb) { - if (!isValidId(channel)) { return void cb('INVALID_CHAN'); } + if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } if (channel.length !== 32) { return void cb('INVALID_CHAN'); } var done = false; @@ -1172,7 +764,7 @@ var writePrivateMessage = function (Env, args, nfwssCtx, cb) { if (!msg) { return void cb("INVALID_MESSAGE"); } // don't support anything except regular channels - if (!isValidId(channelId) || channelId.length !== 32) { + if (!Core.isValidId(channelId) || channelId.length !== 32) { return void cb("INVALID_CHAN"); } @@ -1270,7 +862,7 @@ var upload_status = function (Env, safeKey, filesize, _cb) { // FIXME FILES }).nThen(function () { // if yuo're here then there are no pending uploads // check if you have space in your quota to upload something of this size - getFreeSpace(Env, safeKey, function (e, free) { + Pinning.getFreeSpace(Env, safeKey, function (e, free) { if (e) { return void cb(e); } if (filesize >= free) { return cb('NOT_ENOUGH_SPACE'); } cb(void 0, false); @@ -1300,6 +892,8 @@ RPC.create = function (config, cb) { limits: {}, admins: [], sessionExpirationInterval: undefined, + Log: Log, + WARN: WARN, }; try { @@ -1344,7 +938,7 @@ RPC.create = function (config, cb) { break; } case 'GET_FILE_SIZE': - return void getFileSize(Env, msg[1], function (e, size) { + return void Pinning.getFileSize(Env, msg[1], function (e, size) { WARN(e, msg[1]); respond(e, [null, size, null]); }); @@ -1354,7 +948,7 @@ RPC.create = function (config, cb) { respond(e, [null, data, null]); }); case 'GET_MULTIPLE_FILE_SIZE': - return void getMultipleFileSize(Env, msg[1], function (e, dict) { + return void Pinning.getMultipleFileSize(Env, msg[1], function (e, dict) { if (e) { WARN(e, dict); return respond(e); @@ -1414,7 +1008,7 @@ RPC.create = function (config, cb) { // make sure a user object is initialized in the cookie jar if (publicKey) { - getSession(Sessions, publicKey); + Core.getSession(Sessions, publicKey); } else { Log.debug("NO_PUBLIC_KEY_PROVIDED", publicKey); } @@ -1471,38 +1065,33 @@ RPC.create = function (config, cb) { switch (msg[0]) { case 'COOKIE': return void Respond(void 0); case 'RESET': - return resetUserPins(Env, safeKey, msg[1], function (e, hash) { + return Pinning.resetUserPins(Env, safeKey, msg[1], function (e, hash) { //WARN(e, hash); return void Respond(e, hash); }); case 'PIN': - return pinChannel(Env, safeKey, msg[1], function (e, hash) { + return Pinning.pinChannel(Env, safeKey, msg[1], function (e, hash) { WARN(e, hash); Respond(e, hash); }); case 'UNPIN': - return unpinChannel(Env, safeKey, msg[1], function (e, hash) { + return Pinning.unpinChannel(Env, safeKey, msg[1], function (e, hash) { WARN(e, hash); Respond(e, hash); }); case 'GET_HASH': - return void getHash(Env, safeKey, function (e, hash) { + return void Pinning.getHash(Env, safeKey, function (e, hash) { WARN(e, hash); Respond(e, hash); }); case 'GET_TOTAL_SIZE': // TODO cache this, since it will get called quite a bit - return getTotalSize(Env, safeKey, function (e, size) { + return Pinning.getTotalSize(Env, safeKey, function (e, size) { if (e) { WARN(e, safeKey); return void Respond(e); } Respond(e, size); }); - case 'GET_FILE_SIZE': - return void getFileSize(Env, msg[1], function (e, size) { - WARN(e, msg[1]); - Respond(e, size); - }); case 'UPDATE_LIMITS': return void updateLimits(Env, config, safeKey, function (e, limit) { if (e) { @@ -1512,21 +1101,13 @@ RPC.create = function (config, cb) { Respond(void 0, limit); }); case 'GET_LIMIT': - return void getLimit(Env, safeKey, function (e, limit) { + return void Pinning.getLimit(Env, safeKey, function (e, limit) { if (e) { WARN(e, limit); return void Respond(e); } Respond(void 0, limit); }); - case 'GET_MULTIPLE_FILE_SIZE': - return void getMultipleFileSize(Env, msg[1], function (e, dict) { - if (e) { - WARN(e, dict); - return void Respond(e); - } - Respond(void 0, dict); - }); case 'EXPIRE_SESSION': return void setTimeout(function () { expireSession(Sessions, safeKey); @@ -1549,12 +1130,12 @@ RPC.create = function (config, cb) { Respond(void 0, 'OK'); }); case 'REMOVE_PINS': - return void removePins(Env, safeKey, function (e) { + return void Pinning.removePins(Env, safeKey, function (e) { if (e) { return void Respond(e); } Respond(void 0, "OK"); }); case 'TRIM_PINS': - return void trimPins(Env, safeKey, function (e) { + return void Pinning.trimPins(Env, safeKey, function (e) { if (e) { return void Respond(e); } Respond(void 0, "OK"); }); @@ -1568,7 +1149,7 @@ RPC.create = function (config, cb) { return void upload_status(Env, safeKey, filesize, function (e, yes) { if (!e && !yes) { // no pending uploads, set the new size - var user = getSession(Sessions, safeKey); + var user = Core.getSession(Sessions, safeKey); user.pendingUploadSize = filesize; user.currentUploadSize = 0; } @@ -1665,7 +1246,7 @@ RPC.create = function (config, cb) { blobStagingPath: config.blobStagingPath, archivePath: config.archivePath, getSession: function (safeKey) { - return getSession(Sessions, safeKey); + return Core.getSession(Sessions, safeKey); }, }, w(function (err, blob) { if (err) { throw new Error(err); } @@ -1677,6 +1258,6 @@ RPC.create = function (config, cb) { // XXX allow for graceful shutdown Env.sessionExpirationInterval = setInterval(function () { expireSessions(Sessions); - }, SESSION_EXPIRATION_TIME); + }, Core.SESSION_EXPIRATION_TIME); }); }; From c765362744b4cfb143fc29ac684b5d79be4c6436 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 13:06:46 -0500 Subject: [PATCH 029/172] move more rpc functionality into modules --- lib/commands/core.js | 153 ++++++++++++++++- lib/commands/pin-rpc.js | 65 ++++++++ lib/commands/quota.js | 111 +++++++++++++ lib/rpc.js | 359 +++------------------------------------- 4 files changed, 346 insertions(+), 342 deletions(-) create mode 100644 lib/commands/quota.js diff --git a/lib/commands/core.js b/lib/commands/core.js index 7e34232cb..0bf7ff542 100644 --- a/lib/commands/core.js +++ b/lib/commands/core.js @@ -1,8 +1,13 @@ /*jshint esversion: 6 */ +/* globals process */ const Core = module.exports; const Util = require("../common-util"); const escapeKeyCharacters = Util.escapeKeyCharacters; +/* Use Nacl for checking signatures of messages */ +const Nacl = require("tweetnacl/nacl-fast"); + + Core.DEFAULT_LIMIT = 50 * 1024 * 1024; Core.SESSION_EXPIRATION_TIME = 60 * 1000; @@ -16,6 +21,30 @@ var makeToken = Core.makeToken = function () { .toString(16); }; +Core.makeCookie = function (token) { + var time = (+new Date()); + time -= time % 5000; + + return [ + time, + process.pid, + token + ]; +}; + +var parseCookie = function (cookie) { + if (!(cookie && cookie.split)) { return null; } + + var parts = cookie.split('|'); + if (parts.length !== 3) { return null; } + + var c = {}; + c.time = new Date(parts[0]); + c.pid = Number(parts[1]); + c.seq = parts[2]; + return c; +}; + Core.getSession = function (Sessions, key) { var safeKey = escapeKeyCharacters(key); if (Sessions[safeKey]) { @@ -30,13 +59,123 @@ Core.getSession = function (Sessions, key) { return user; }; +Core.expireSession = function (Sessions, key) { + var session = Sessions[key]; + if (!session) { return; } + if (session.blobstage) { + session.blobstage.close(); + } + delete Sessions[key]; +}; + +var isTooOld = function (time, now) { + return (now - time) > 300000; +}; + +Core.expireSessions = function (Sessions) { + var now = +new Date(); + Object.keys(Sessions).forEach(function (key) { + var session = Sessions[key]; + if (session && isTooOld(session.atime, now)) { + Core.expireSession(Sessions, key); + } + }); +}; + +var addTokenForKey = function (Sessions, publicKey, token) { + if (!Sessions[publicKey]) { throw new Error('undefined user'); } + + var user = Core.getSession(Sessions, publicKey); + user.tokens.push(token); + user.atime = +new Date(); + if (user.tokens.length > 2) { user.tokens.shift(); } +}; + +Core.isValidCookie = function (Sessions, publicKey, cookie) { + var parsed = parseCookie(cookie); + if (!parsed) { return false; } + + var now = +new Date(); + + if (!parsed.time) { return false; } + if (isTooOld(parsed.time, now)) { + return false; + } + + // different process. try harder + if (process.pid !== parsed.pid) { + return false; + } + + var user = Core.getSession(Sessions, publicKey); + if (!user) { return false; } + + var idx = user.tokens.indexOf(parsed.seq); + if (idx === -1) { return false; } + + if (idx > 0) { + // make a new token + addTokenForKey(Sessions, publicKey, Core.makeToken()); + } + + return true; +}; + +Core.checkSignature = function (Env, signedMsg, signature, publicKey) { + if (!(signedMsg && publicKey)) { return false; } + + var signedBuffer; + var pubBuffer; + var signatureBuffer; + + try { + signedBuffer = Nacl.util.decodeUTF8(signedMsg); + } catch (e) { + Env.Log.error('INVALID_SIGNED_BUFFER', signedMsg); + return null; + } + + try { + pubBuffer = Nacl.util.decodeBase64(publicKey); + } catch (e) { + return false; + } + + try { + signatureBuffer = Nacl.util.decodeBase64(signature); + } catch (e) { + return false; + } + + if (pubBuffer.length !== 32) { + Env.Log.error('PUBLIC_KEY_LENGTH', publicKey); + return false; + } + + if (signatureBuffer.length !== 64) { + return false; + } + + return Nacl.sign.detached.verify(signedBuffer, signatureBuffer, pubBuffer); +}; + +// E_NO_OWNERS +Core.hasOwners = function (metadata) { + return Boolean(metadata && Array.isArray(metadata.owners)); +}; + +Core.hasPendingOwners = function (metadata) { + return Boolean(metadata && Array.isArray(metadata.pending_owners)); +}; + +// INSUFFICIENT_PERMISSIONS +Core.isOwner = function (metadata, unsafeKey) { + return metadata.owners.indexOf(unsafeKey) !== -1; +}; + +Core.isPendingOwner = function (metadata, unsafeKey) { + return metadata.pending_owners.indexOf(unsafeKey) !== -1; +}; -// getChannelList -// getSession -// getHash -// getMultipleFileSize -// sumChannelSizes -// getFreeSpace -// getLimit diff --git a/lib/commands/pin-rpc.js b/lib/commands/pin-rpc.js index 3412f329e..16998b8b5 100644 --- a/lib/commands/pin-rpc.js +++ b/lib/commands/pin-rpc.js @@ -8,6 +8,8 @@ const Pinning = module.exports; const Nacl = require("tweetnacl/nacl-fast"); const Util = require("../common-util"); const nThen = require("nthen"); +const Saferphore = require("saferphore"); +const Pinned = require('../../scripts/pinned'); //const escapeKeyCharacters = Util.escapeKeyCharacters; const unescapeKeyCharacters = Util.unescapeKeyCharacters; @@ -397,3 +399,66 @@ Pinning.getFileSize = function (Env, channel, _cb) { }); }; +/* accepts a list, and returns a sublist of channel or file ids which seem + to have been deleted from the server (file size 0) + + we might consider that we should only say a file is gone if fs.stat returns + ENOENT, but for now it's simplest to just rely on getFileSize... +*/ +Pinning.getDeletedPads = function (Env, channels, cb) { + if (!Array.isArray(channels)) { return cb('INVALID_LIST'); } + var L = channels.length; + + var sem = Saferphore.create(10); + var absentees = []; + + var job = function (channel, wait) { + return function (give) { + Pinning.getFileSize(Env, channel, wait(give(function (e, size) { + if (e) { return; } + if (size === 0) { absentees.push(channel); } + }))); + }; + }; + + nThen(function (w) { + for (var i = 0; i < L; i++) { + sem.take(job(channels[i], w)); + } + }).nThen(function () { + cb(void 0, absentees); + }); +}; + +// inform that the +Pinning.loadChannelPins = function (Env) { + Pinned.load(function (err, data) { + if (err) { + Env.Log.error("LOAD_CHANNEL_PINS", err); + + // FIXME not sure what should be done here instead + Env.pinnedPads = {}; + Env.evPinnedPadsReady.fire(); + return; + } + + + Env.pinnedPads = data; + Env.evPinnedPadsReady.fire(); + }, { + pinPath: Env.paths.pin, + }); +}; + +Pinning.isChannelPinned = function (Env, channel, cb) { + Env.evPinnedPadsReady.reg(() => { + if (Env.pinnedPads[channel] && Object.keys(Env.pinnedPads[channel]).length) { + cb(true); + } else { + delete Env.pinnedPads[channel]; + cb(false); + } + }); +}; + + diff --git a/lib/commands/quota.js b/lib/commands/quota.js new file mode 100644 index 000000000..e7df14364 --- /dev/null +++ b/lib/commands/quota.js @@ -0,0 +1,111 @@ +/*jshint esversion: 6 */ +/* globals Buffer*/ +const Quota = module.exports; + +const Core = require("./commands/core"); +const Util = require("../common-util"); +const Package = require('../../package.json'); +const Https = require("https"); + +Quota.applyCustomLimits = function (Env, config) { + var isLimit = function (o) { + var valid = o && typeof(o) === 'object' && + typeof(o.limit) === 'number' && + typeof(o.plan) === 'string' && + typeof(o.note) === 'string'; + return valid; + }; + + // read custom limits from the config + var customLimits = (function (custom) { + var limits = {}; + Object.keys(custom).forEach(function (k) { + k.replace(/\/([^\/]+)$/, function (all, safeKey) { + var id = Util.unescapeKeyCharacters(safeKey || ''); + limits[id] = custom[k]; + return ''; + }); + }); + return limits; + }(config.customLimits || {})); + + Object.keys(customLimits).forEach(function (k) { + if (!isLimit(customLimits[k])) { return; } + Env.limits[k] = customLimits[k]; + }); +}; + +// The limits object contains storage limits for all the publicKey that have paid +// To each key is associated an object containing the 'limit' value and a 'note' explaining that limit +Quota.updateLimits = function (Env, config, publicKey, cb) { // FIXME BATCH?S + + if (config.adminEmail === false) { + Quota.applyCustomLimits(Env, config); + if (config.allowSubscriptions === false) { return; } + throw new Error("allowSubscriptions must be false if adminEmail is false"); + } + if (typeof cb !== "function") { cb = function () {}; } + + var defaultLimit = typeof(config.defaultStorageLimit) === 'number'? + config.defaultStorageLimit: Core.DEFAULT_LIMIT; + + var userId; + if (publicKey) { + userId = Util.unescapeKeyCharacters(publicKey); + } + + var body = JSON.stringify({ + domain: config.myDomain, + subdomain: config.mySubdomain || null, + adminEmail: config.adminEmail, + version: Package.version + }); + var options = { + host: 'accounts.cryptpad.fr', + path: '/api/getauthorized', + method: 'POST', + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(body) + } + }; + + var req = Https.request(options, function (response) { + if (!('' + response.statusCode).match(/^2\d\d$/)) { + return void cb('SERVER ERROR ' + response.statusCode); + } + var str = ''; + + response.on('data', function (chunk) { + str += chunk; + }); + + response.on('end', function () { + try { + var json = JSON.parse(str); + Env.limits = json; + Quota.applyCustomLimits(Env, config); + + var l; + if (userId) { + var limit = Env.limits[userId]; + l = limit && typeof limit.limit === "number" ? + [limit.limit, limit.plan, limit.note] : [defaultLimit, '', '']; + } + cb(void 0, l); + } catch (e) { + cb(e); + } + }); + }); + + req.on('error', function (e) { + Quota.applyCustomLimits(Env, config); + if (!config.domain) { return cb(); } + cb(e); + }); + + req.end(body); +}; + + diff --git a/lib/rpc.js b/lib/rpc.js index a064ebe64..cbaad1ee2 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -1,32 +1,26 @@ -/*@flow*/ /*jshint esversion: 6 */ /* Use Nacl for checking signatures of messages */ var Nacl = require("tweetnacl/nacl-fast"); /* globals Buffer*/ -/* globals process */ var Fs = require("fs"); var Fse = require("fs-extra"); var Path = require("path"); -var Https = require("https"); -const Package = require('../package.json'); -const Pinned = require('../scripts/pinned'); -const Saferphore = require("saferphore"); const nThen = require("nthen"); const Meta = require("./metadata"); const WriteQueue = require("./write-queue"); const BatchRead = require("./batch-read"); -const Core = require("./commands/core"); const Util = require("./common-util"); const escapeKeyCharacters = Util.escapeKeyCharacters; -const unescapeKeyCharacters = Util.unescapeKeyCharacters; const mkEvent = Util.mkEvent; +const Core = require("./commands/core"); const Admin = require("./commands/admin-rpc"); const Pinning = require("./commands/pin-rpc"); +const Quota = require("./commands/quota"); var RPC = module.exports; @@ -45,130 +39,6 @@ var WARN = function (e, output) { } }; -var makeCookie = function (token) { - var time = (+new Date()); - time -= time % 5000; - - return [ - time, - process.pid, - token - ]; -}; - -var parseCookie = function (cookie) { - if (!(cookie && cookie.split)) { return null; } - - var parts = cookie.split('|'); - if (parts.length !== 3) { return null; } - - var c = {}; - c.time = new Date(parts[0]); - c.pid = Number(parts[1]); - c.seq = parts[2]; - return c; -}; - -var isTooOld = function (time, now) { - return (now - time) > 300000; -}; - -var expireSession = function (Sessions, key) { - var session = Sessions[key]; - if (!session) { return; } - if (session.blobstage) { - session.blobstage.close(); - } - delete Sessions[key]; -}; - -var expireSessions = function (Sessions) { - var now = +new Date(); - Object.keys(Sessions).forEach(function (key) { - var session = Sessions[key]; - if (session && isTooOld(session.atime, now)) { - expireSession(Sessions, key); - } - }); -}; - -var addTokenForKey = function (Sessions, publicKey, token) { - if (!Sessions[publicKey]) { throw new Error('undefined user'); } - - var user = Core.getSession(Sessions, publicKey); - user.tokens.push(token); - user.atime = +new Date(); - if (user.tokens.length > 2) { user.tokens.shift(); } -}; - -var isValidCookie = function (Sessions, publicKey, cookie) { - var parsed = parseCookie(cookie); - if (!parsed) { return false; } - - var now = +new Date(); - - if (!parsed.time) { return false; } - if (isTooOld(parsed.time, now)) { - return false; - } - - // different process. try harder - if (process.pid !== parsed.pid) { - return false; - } - - var user = Core.getSession(Sessions, publicKey); - if (!user) { return false; } - - var idx = user.tokens.indexOf(parsed.seq); - if (idx === -1) { return false; } - - if (idx > 0) { - // make a new token - addTokenForKey(Sessions, publicKey, Core.makeToken()); - } - - return true; -}; - -var checkSignature = function (signedMsg, signature, publicKey) { - if (!(signedMsg && publicKey)) { return false; } - - var signedBuffer; - var pubBuffer; - var signatureBuffer; - - try { - signedBuffer = Nacl.util.decodeUTF8(signedMsg); - } catch (e) { - Log.error('INVALID_SIGNED_BUFFER', signedMsg); - return null; - } - - try { - pubBuffer = Nacl.util.decodeBase64(publicKey); - } catch (e) { - return false; - } - - try { - signatureBuffer = Nacl.util.decodeBase64(signature); - } catch (e) { - return false; - } - - if (pubBuffer.length !== 32) { - Log.error('PUBLIC_KEY_LENGTH', publicKey); - return false; - } - - if (signatureBuffer.length !== 64) { - return false; - } - - return Nacl.sign.detached.verify(signedBuffer, signatureBuffer, pubBuffer); -}; - const batchMetadata = BatchRead("GET_METADATA"); var getMetadata = function (Env, channel, cb) { if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } @@ -188,24 +58,6 @@ var getMetadata = function (Env, channel, cb) { }); }; -// E_NO_OWNERS -var hasOwners = function (metadata) { - return Boolean(metadata && Array.isArray(metadata.owners)); -}; - -var hasPendingOwners = function (metadata) { - return Boolean(metadata && Array.isArray(metadata.pending_owners)); -}; - -// INSUFFICIENT_PERMISSIONS -var isOwner = function (metadata, unsafeKey) { - return metadata.owners.indexOf(unsafeKey) !== -1; -}; - -var isPendingOwner = function (metadata, unsafeKey) { - return metadata.pending_owners.indexOf(unsafeKey) !== -1; -}; - /* setMetadata - write a new line to the metadata log if a valid command is provided - data is an object: { @@ -228,7 +80,7 @@ var setMetadata = function (Env, data, unsafeKey, cb) { cb(err); return void next(); } - if (!hasOwners(metadata)) { + if (!Core.hasOwners(metadata)) { cb('E_NO_OWNERS'); return void next(); } @@ -243,9 +95,9 @@ var setMetadata = function (Env, data, unsafeKey, cb) { // Confirm that the channel is owned by the user in question // or the user is accepting a pending ownership offer - if (hasPendingOwners(metadata) && - isPendingOwner(metadata, unsafeKey) && - !isOwner(metadata, unsafeKey)) { + if (Core.hasPendingOwners(metadata) && + Core.isPendingOwner(metadata, unsafeKey) && + !Core.isOwner(metadata, unsafeKey)) { // If you are a pending owner, make sure you can only add yourelf as an owner if ((command !== 'ADD_OWNERS' && command !== 'RM_PENDING_OWNERS') @@ -258,7 +110,7 @@ var setMetadata = function (Env, data, unsafeKey, cb) { // FIXME wacky fallthrough is hard to read // we could pass this off to a writeMetadataCommand function // and make the flow easier to follow - } else if (!isOwner(metadata, unsafeKey)) { + } else if (!Core.isOwner(metadata, unsafeKey)) { cb('INSUFFICIENT_PERMISSIONS'); return void next(); } @@ -291,169 +143,6 @@ var setMetadata = function (Env, data, unsafeKey, cb) { }); }; -/* accepts a list, and returns a sublist of channel or file ids which seem - to have been deleted from the server (file size 0) - - we might consider that we should only say a file is gone if fs.stat returns - ENOENT, but for now it's simplest to just rely on getFileSize... -*/ -var getDeletedPads = function (Env, channels, cb) { - if (!Array.isArray(channels)) { return cb('INVALID_LIST'); } - var L = channels.length; - - var sem = Saferphore.create(10); - var absentees = []; - - var job = function (channel, wait) { - return function (give) { - Pinning.getFileSize(Env, channel, wait(give(function (e, size) { - if (e) { return; } - if (size === 0) { absentees.push(channel); } - }))); - }; - }; - - nThen(function (w) { - for (var i = 0; i < L; i++) { - sem.take(job(channels[i], w)); - } - }).nThen(function () { - cb(void 0, absentees); - }); -}; - -var applyCustomLimits = function (Env, config) { - var isLimit = function (o) { - var valid = o && typeof(o) === 'object' && - typeof(o.limit) === 'number' && - typeof(o.plan) === 'string' && - typeof(o.note) === 'string'; - return valid; - }; - - // read custom limits from the config - var customLimits = (function (custom) { - var limits = {}; - Object.keys(custom).forEach(function (k) { - k.replace(/\/([^\/]+)$/, function (all, safeKey) { - var id = unescapeKeyCharacters(safeKey || ''); - limits[id] = custom[k]; - return ''; - }); - }); - return limits; - }(config.customLimits || {})); - - Object.keys(customLimits).forEach(function (k) { - if (!isLimit(customLimits[k])) { return; } - Env.limits[k] = customLimits[k]; - }); -}; - -// The limits object contains storage limits for all the publicKey that have paid -// To each key is associated an object containing the 'limit' value and a 'note' explaining that limit -var updateLimits = function (Env, config, publicKey, cb /*:(?string, ?any[])=>void*/) { // FIXME BATCH? - - if (config.adminEmail === false) { - applyCustomLimits(Env, config); - if (config.allowSubscriptions === false) { return; } - throw new Error("allowSubscriptions must be false if adminEmail is false"); - } - if (typeof cb !== "function") { cb = function () {}; } - - var defaultLimit = typeof(config.defaultStorageLimit) === 'number'? - config.defaultStorageLimit: Core.DEFAULT_LIMIT; - - var userId; - if (publicKey) { - userId = unescapeKeyCharacters(publicKey); - } - - var body = JSON.stringify({ - domain: config.myDomain, - subdomain: config.mySubdomain || null, - adminEmail: config.adminEmail, - version: Package.version - }); - var options = { - host: 'accounts.cryptpad.fr', - path: '/api/getauthorized', - method: 'POST', - headers: { - "Content-Type": "application/json", - "Content-Length": Buffer.byteLength(body) - } - }; - - var req = Https.request(options, function (response) { - if (!('' + response.statusCode).match(/^2\d\d$/)) { - return void cb('SERVER ERROR ' + response.statusCode); - } - var str = ''; - - response.on('data', function (chunk) { - str += chunk; - }); - - response.on('end', function () { - try { - var json = JSON.parse(str); - Env.limits = json; - applyCustomLimits(Env, config); - - var l; - if (userId) { - var limit = Env.limits[userId]; - l = limit && typeof limit.limit === "number" ? - [limit.limit, limit.plan, limit.note] : [defaultLimit, '', '']; - } - cb(void 0, l); - } catch (e) { - cb(e); - } - }); - }); - - req.on('error', function (e) { - applyCustomLimits(Env, config); - if (!config.domain) { return cb(); } - cb(e); - }); - - req.end(body); -}; - -// inform that the -var loadChannelPins = function (Env) { - Pinned.load(function (err, data) { - if (err) { - Log.error("LOAD_CHANNEL_PINS", err); - - // FIXME not sure what should be done here instead - Env.pinnedPads = {}; - Env.evPinnedPadsReady.fire(); - return; - } - - - Env.pinnedPads = data; - Env.evPinnedPadsReady.fire(); - }, { - pinPath: Env.paths.pin, - }); -}; - -var isChannelPinned = function (Env, channel, cb) { - Env.evPinnedPadsReady.reg(() => { - if (Env.pinnedPads[channel] && Object.keys(Env.pinnedPads[channel]).length) { - cb(true); - } else { - delete Env.pinnedPads[channel]; - cb(false); - } - }); -}; - var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { if (typeof(channelId) !== 'string' || channelId.length !== 32) { return cb('INVALID_ARGUMENTS'); @@ -461,9 +150,9 @@ var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } - if (!hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } + if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } // Confirm that the channel is owned by the user in question - if (!isOwner(metadata, unsafeKey)) { + if (!Core.isOwner(metadata, unsafeKey)) { return void cb('INSUFFICIENT_PERMISSIONS'); } return void Env.msgStore.clearChannel(channelId, function (e) { @@ -520,8 +209,8 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } - if (!hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } - if (!isOwner(metadata, unsafeKey)) { + if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } + if (!Core.isOwner(metadata, unsafeKey)) { return void cb('INSUFFICIENT_PERMISSIONS'); } // temporarily archive the file @@ -540,11 +229,11 @@ var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { nThen(function (w) { getMetadata(Env, channelId, w(function (err, metadata) { if (err) { return void cb(err); } - if (!hasOwners(metadata)) { + if (!Core.hasOwners(metadata)) { w.abort(); return void cb('E_NO_OWNERS'); } - if (!isOwner(metadata, unsafeKey)) { + if (!Core.isOwner(metadata, unsafeKey)) { w.abort(); return void cb("INSUFFICIENT_PERMISSIONS"); } @@ -956,7 +645,7 @@ RPC.create = function (config, cb) { respond(e, [null, dict, null]); }); case 'GET_DELETED_PADS': - return void getDeletedPads(Env, msg[1], function (e, list) { + return void Pinning.getDeletedPads(Env, msg[1], function (e, list) { if (e) { WARN(e, msg[1]); return respond(e); @@ -964,7 +653,7 @@ RPC.create = function (config, cb) { respond(e, [null, list, null]); }); case 'IS_CHANNEL_PINNED': - return void isChannelPinned(Env, msg[1], function (isPinned) { + return void Pinning.isChannelPinned(Env, msg[1], function (isPinned) { respond(null, [null, isPinned, null]); }); case 'IS_NEW_CHANNEL': @@ -1014,7 +703,7 @@ RPC.create = function (config, cb) { } var cookie = msg[0]; - if (!isValidCookie(Sessions, publicKey, cookie)) { + if (!Core.isValidCookie(Sessions, publicKey, cookie)) { // no cookie is fine if the RPC is to get a cookie if (msg[1] !== 'COOKIE') { return void respond('NO_COOKIE'); @@ -1028,7 +717,7 @@ RPC.create = function (config, cb) { } if (isAuthenticatedCall(msg[1])) { - if (checkSignature(serialized, signature, publicKey) !== true) { + if (Core.checkSignature(Env, serialized, signature, publicKey) !== true) { return void respond("INVALID_SIGNATURE_OR_PUBLIC_KEY"); } } else if (msg[1] !== 'UPLOAD') { @@ -1052,7 +741,7 @@ RPC.create = function (config, cb) { var Respond = function (e, msg) { var session = Sessions[safeKey]; var token = session? session.tokens.slice(-1)[0]: ''; - var cookie = makeCookie(token).join('|'); + var cookie = Core.makeCookie(token).join('|'); respond(e ? String(e): e, [cookie].concat(typeof(msg) !== 'undefined' ?msg: [])); }; @@ -1093,7 +782,7 @@ RPC.create = function (config, cb) { Respond(e, size); }); case 'UPDATE_LIMITS': - return void updateLimits(Env, config, safeKey, function (e, limit) { + return void Quota.updateLimits(Env, config, safeKey, function (e, limit) { if (e) { WARN(e, limit); return void Respond(e); @@ -1110,7 +799,7 @@ RPC.create = function (config, cb) { }); case 'EXPIRE_SESSION': return void setTimeout(function () { - expireSession(Sessions, safeKey); + Core.expireSession(Sessions, safeKey); Respond(void 0, "OK"); }); case 'CLEAR_OWNED_CHANNEL': @@ -1223,17 +912,17 @@ RPC.create = function (config, cb) { }; var updateLimitDaily = function () { - updateLimits(Env, config, undefined, function (e) { + Quota.updateLimits(Env, config, undefined, function (e) { if (e) { WARN('limitUpdate', e); } }); }; - applyCustomLimits(Env, config); + Quota.applyCustomLimits(Env, config); updateLimitDaily(); setInterval(updateLimitDaily, 24*3600*1000); - loadChannelPins(Env); + Pinning.loadChannelPins(Env); nThen(function (w) { Store.create({ @@ -1257,7 +946,7 @@ RPC.create = function (config, cb) { // expire old sessions once per minute // XXX allow for graceful shutdown Env.sessionExpirationInterval = setInterval(function () { - expireSessions(Sessions); + Core.expireSessions(Sessions); }, Core.SESSION_EXPIRATION_TIME); }); }; From bb7e8e4512108bd520ff00b9e6fd737221b095ad Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 13:14:26 -0500 Subject: [PATCH 030/172] move login block functionality into its own rpc module --- lib/commands/block.js | 172 +++++++++++++++++++++++++++++++++++++++++ lib/rpc.js | 175 +----------------------------------------- 2 files changed, 175 insertions(+), 172 deletions(-) create mode 100644 lib/commands/block.js diff --git a/lib/commands/block.js b/lib/commands/block.js new file mode 100644 index 000000000..90837745e --- /dev/null +++ b/lib/commands/block.js @@ -0,0 +1,172 @@ +/*jshint esversion: 6 */ +/* globals Buffer*/ +var Block = module.exports; + +const Fs = require("fs"); +const Fse = require("fs-extra"); +const Path = require("path"); +const Nacl = require("tweetnacl/nacl-fast"); +const nThen = require("nthen"); + +const Util = require("../common-util"); + +/* + We assume that the server is secured against MitM attacks + via HTTPS, and that malicious actors do not have code execution + capabilities. If they do, we have much more serious problems. + + The capability to replay a block write or remove results in either + a denial of service for the user whose block was removed, or in the + case of a write, a rollback to an earlier password. + + Since block modification is destructive, this can result in loss + of access to the user's drive. + + So long as the detached signature is never observed by a malicious + party, and the server discards it after proof of knowledge, replays + are not possible. However, this precludes verification of the signature + at a later time. + + Despite this, an integrity check is still possible by the original + author of the block, since we assume that the block will have been + encrypted with xsalsa20-poly1305 which is authenticated. +*/ +Block.validateLoginBlock = function (Env, publicKey, signature, block, cb) { // FIXME BLOCKS + // convert the public key to a Uint8Array and validate it + if (typeof(publicKey) !== 'string') { return void cb('E_INVALID_KEY'); } + + var u8_public_key; + try { + u8_public_key = Nacl.util.decodeBase64(publicKey); + } catch (e) { + return void cb('E_INVALID_KEY'); + } + + var u8_signature; + try { + u8_signature = Nacl.util.decodeBase64(signature); + } catch (e) { + Env.Log.error('INVALID_BLOCK_SIGNATURE', e); + return void cb('E_INVALID_SIGNATURE'); + } + + // convert the block to a Uint8Array + var u8_block; + try { + u8_block = Nacl.util.decodeBase64(block); + } catch (e) { + return void cb('E_INVALID_BLOCK'); + } + + // take its hash + var hash = Nacl.hash(u8_block); + + // validate the signature against the hash of the content + var verified = Nacl.sign.detached.verify(hash, u8_signature, u8_public_key); + + // existing authentication ensures that users cannot replay old blocks + + // call back with (err) if unsuccessful + if (!verified) { return void cb("E_COULD_NOT_VERIFY"); } + + return void cb(null, u8_block); +}; + +var createLoginBlockPath = function (Env, publicKey) { // FIXME BLOCKS + // prepare publicKey to be used as a file name + var safeKey = Util.escapeKeyCharacters(publicKey); + + // validate safeKey + if (typeof(safeKey) !== 'string') { + return; + } + + // derive the full path + // /home/cryptpad/cryptpad/block/fg/fg32kefksjdgjkewrjksdfksjdfsdfskdjfsfd + return Path.join(Env.paths.block, safeKey.slice(0, 2), safeKey); +}; + +Block.writeLoginBlock = function (Env, msg, cb) { // FIXME BLOCKS + //console.log(msg); + var publicKey = msg[0]; + var signature = msg[1]; + var block = msg[2]; + + Block.validateLoginBlock(Env, publicKey, signature, block, function (e, validatedBlock) { + if (e) { return void cb(e); } + if (!(validatedBlock instanceof Uint8Array)) { return void cb('E_INVALID_BLOCK'); } + + // derive the filepath + var path = createLoginBlockPath(Env, publicKey); + + // make sure the path is valid + if (typeof(path) !== 'string') { + return void cb('E_INVALID_BLOCK_PATH'); + } + + var parsed = Path.parse(path); + if (!parsed || typeof(parsed.dir) !== 'string') { + return void cb("E_INVALID_BLOCK_PATH_2"); + } + + nThen(function (w) { + // make sure the path to the file exists + Fse.mkdirp(parsed.dir, w(function (e) { + if (e) { + w.abort(); + cb(e); + } + })); + }).nThen(function () { + // actually write the block + + // flow is dumb and I need to guard against this which will never happen + /*:: if (typeof(validatedBlock) === 'undefined') { throw new Error('should never happen'); } */ + /*:: if (typeof(path) === 'undefined') { throw new Error('should never happen'); } */ + Fs.writeFile(path, Buffer.from(validatedBlock), { encoding: "binary", }, function (err) { + if (err) { return void cb(err); } + cb(); + }); + }); + }); +}; + +/* + When users write a block, they upload the block, and provide + a signature proving that they deserve to be able to write to + the location determined by the public key. + + When removing a block, there is nothing to upload, but we need + to sign something. Since the signature is considered sensitive + information, we can just sign some constant and use that as proof. + +*/ +Block.removeLoginBlock = function (Env, msg, cb) { // FIXME BLOCKS + var publicKey = msg[0]; + var signature = msg[1]; + var block = Nacl.util.decodeUTF8('DELETE_BLOCK'); // clients and the server will have to agree on this constant + + Block.validateLoginBlock(Env, publicKey, signature, block, function (e /*::, validatedBlock */) { + if (e) { return void cb(e); } + // derive the filepath + var path = createLoginBlockPath(Env, publicKey); + + // make sure the path is valid + if (typeof(path) !== 'string') { + return void cb('E_INVALID_BLOCK_PATH'); + } + + // FIXME COLDSTORAGE + Fs.unlink(path, function (err) { + Env.Log.info('DELETION_BLOCK_BY_OWNER_RPC', { + publicKey: publicKey, + path: path, + status: err? String(err): 'SUCCESS', + }); + + if (err) { return void cb(err); } + cb(); + }); + }); +}; + diff --git a/lib/rpc.js b/lib/rpc.js index cbaad1ee2..3785b20a1 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -1,13 +1,4 @@ /*jshint esversion: 6 */ -/* Use Nacl for checking signatures of messages */ -var Nacl = require("tweetnacl/nacl-fast"); - -/* globals Buffer*/ - -var Fs = require("fs"); - -var Fse = require("fs-extra"); -var Path = require("path"); const nThen = require("nthen"); const Meta = require("./metadata"); const WriteQueue = require("./write-queue"); @@ -21,6 +12,7 @@ const Core = require("./commands/core"); const Admin = require("./commands/admin-rpc"); const Pinning = require("./commands/pin-rpc"); const Quota = require("./commands/quota"); +const Block = require("./commands/block"); var RPC = module.exports; @@ -249,167 +241,6 @@ var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { }); }; - -/* - We assume that the server is secured against MitM attacks - via HTTPS, and that malicious actors do not have code execution - capabilities. If they do, we have much more serious problems. - - The capability to replay a block write or remove results in either - a denial of service for the user whose block was removed, or in the - case of a write, a rollback to an earlier password. - - Since block modification is destructive, this can result in loss - of access to the user's drive. - - So long as the detached signature is never observed by a malicious - party, and the server discards it after proof of knowledge, replays - are not possible. However, this precludes verification of the signature - at a later time. - - Despite this, an integrity check is still possible by the original - author of the block, since we assume that the block will have been - encrypted with xsalsa20-poly1305 which is authenticated. -*/ -var validateLoginBlock = function (Env, publicKey, signature, block, cb) { // FIXME BLOCKS - // convert the public key to a Uint8Array and validate it - if (typeof(publicKey) !== 'string') { return void cb('E_INVALID_KEY'); } - - var u8_public_key; - try { - u8_public_key = Nacl.util.decodeBase64(publicKey); - } catch (e) { - return void cb('E_INVALID_KEY'); - } - - var u8_signature; - try { - u8_signature = Nacl.util.decodeBase64(signature); - } catch (e) { - Log.error('INVALID_BLOCK_SIGNATURE', e); - return void cb('E_INVALID_SIGNATURE'); - } - - // convert the block to a Uint8Array - var u8_block; - try { - u8_block = Nacl.util.decodeBase64(block); - } catch (e) { - return void cb('E_INVALID_BLOCK'); - } - - // take its hash - var hash = Nacl.hash(u8_block); - - // validate the signature against the hash of the content - var verified = Nacl.sign.detached.verify(hash, u8_signature, u8_public_key); - - // existing authentication ensures that users cannot replay old blocks - - // call back with (err) if unsuccessful - if (!verified) { return void cb("E_COULD_NOT_VERIFY"); } - - return void cb(null, u8_block); -}; - -var createLoginBlockPath = function (Env, publicKey) { // FIXME BLOCKS - // prepare publicKey to be used as a file name - var safeKey = escapeKeyCharacters(publicKey); - - // validate safeKey - if (typeof(safeKey) !== 'string') { - return; - } - - // derive the full path - // /home/cryptpad/cryptpad/block/fg/fg32kefksjdgjkewrjksdfksjdfsdfskdjfsfd - return Path.join(Env.paths.block, safeKey.slice(0, 2), safeKey); -}; - -var writeLoginBlock = function (Env, msg, cb) { // FIXME BLOCKS - //console.log(msg); - var publicKey = msg[0]; - var signature = msg[1]; - var block = msg[2]; - - validateLoginBlock(Env, publicKey, signature, block, function (e, validatedBlock) { - if (e) { return void cb(e); } - if (!(validatedBlock instanceof Uint8Array)) { return void cb('E_INVALID_BLOCK'); } - - // derive the filepath - var path = createLoginBlockPath(Env, publicKey); - - // make sure the path is valid - if (typeof(path) !== 'string') { - return void cb('E_INVALID_BLOCK_PATH'); - } - - var parsed = Path.parse(path); - if (!parsed || typeof(parsed.dir) !== 'string') { - return void cb("E_INVALID_BLOCK_PATH_2"); - } - - nThen(function (w) { - // make sure the path to the file exists - Fse.mkdirp(parsed.dir, w(function (e) { - if (e) { - w.abort(); - cb(e); - } - })); - }).nThen(function () { - // actually write the block - - // flow is dumb and I need to guard against this which will never happen - /*:: if (typeof(validatedBlock) === 'undefined') { throw new Error('should never happen'); } */ - /*:: if (typeof(path) === 'undefined') { throw new Error('should never happen'); } */ - Fs.writeFile(path, Buffer.from(validatedBlock), { encoding: "binary", }, function (err) { - if (err) { return void cb(err); } - cb(); - }); - }); - }); -}; - -/* - When users write a block, they upload the block, and provide - a signature proving that they deserve to be able to write to - the location determined by the public key. - - When removing a block, there is nothing to upload, but we need - to sign something. Since the signature is considered sensitive - information, we can just sign some constant and use that as proof. - -*/ -var removeLoginBlock = function (Env, msg, cb) { // FIXME BLOCKS - var publicKey = msg[0]; - var signature = msg[1]; - var block = Nacl.util.decodeUTF8('DELETE_BLOCK'); // clients and the server will have to agree on this constant - - validateLoginBlock(Env, publicKey, signature, block, function (e /*::, validatedBlock */) { - if (e) { return void cb(e); } - // derive the filepath - var path = createLoginBlockPath(Env, publicKey); - - // make sure the path is valid - if (typeof(path) !== 'string') { - return void cb('E_INVALID_BLOCK_PATH'); - } - - // FIXME COLDSTORAGE - Fs.unlink(path, function (err) { - Log.info('DELETION_BLOCK_BY_OWNER_RPC', { - publicKey: publicKey, - path: path, - status: err? String(err): 'SUCCESS', - }); - - if (err) { return void cb(err); } - cb(); - }); - }); -}; - var ARRAY_LINE = /^\[/; /* Files can contain metadata but not content @@ -863,7 +694,7 @@ RPC.create = function (config, cb) { Respond(e); }); case 'WRITE_LOGIN_BLOCK': - return void writeLoginBlock(Env, msg[1], function (e) { + return void Block.writeLoginBlock(Env, msg[1], function (e) { if (e) { WARN(e, 'WRITE_LOGIN_BLOCK'); return void Respond(e); @@ -871,7 +702,7 @@ RPC.create = function (config, cb) { Respond(e); }); case 'REMOVE_LOGIN_BLOCK': - return void removeLoginBlock(Env, msg[1], function (e) { + return void Block.removeLoginBlock(Env, msg[1], function (e) { if (e) { WARN(e, 'REMOVE_LOGIN_BLOCK'); return void Respond(e); From c1f222dd6c8084790efea2313f86934270bbc672 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 13:19:40 -0500 Subject: [PATCH 031/172] move metadata commands from rpc to their own module --- lib/commands/metadata.js | 113 +++++++++++++++++++++++++++++++++++++ lib/rpc.js | 118 ++------------------------------------- 2 files changed, 119 insertions(+), 112 deletions(-) create mode 100644 lib/commands/metadata.js diff --git a/lib/commands/metadata.js b/lib/commands/metadata.js new file mode 100644 index 000000000..91b08d8cc --- /dev/null +++ b/lib/commands/metadata.js @@ -0,0 +1,113 @@ +/*jshint esversion: 6 */ +const Data = module.exports; + +const Meta = require("../metadata"); +const BatchRead = require("../batch-read"); +const WriteQueue = require("./write-queue"); +const Core = require("./commands/core"); + +const batchMetadata = BatchRead("GET_METADATA"); +Data.getMetadata = function (Env, channel, cb) { + if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } + if (channel.length !== 32) { return cb("INVALID_CHAN_LENGTH"); } + + batchMetadata(channel, cb, function (done) { + var ref = {}; + var lineHandler = Meta.createLineHandler(ref, Env.Log.error); + + return void Env.msgStore.readChannelMetadata(channel, lineHandler, function (err) { + if (err) { + // stream errors? + return void done(err); + } + done(void 0, ref.meta); + }); + }); +}; + +/* setMetadata + - write a new line to the metadata log if a valid command is provided + - data is an object: { + channel: channelId, + command: metadataCommand (string), + value: value + } +*/ +var queueMetadata = WriteQueue(); +Data.setMetadata = function (Env, data, unsafeKey, cb) { + var channel = data.channel; + var command = data.command; + if (!channel || !Core.isValidId(channel)) { return void cb ('INVALID_CHAN'); } + if (!command || typeof (command) !== 'string') { return void cb ('INVALID_COMMAND'); } + if (Meta.commands.indexOf(command) === -1) { return void('UNSUPPORTED_COMMAND'); } + + queueMetadata(channel, function (next) { + Data.getMetadata(Env, channel, function (err, metadata) { + if (err) { + cb(err); + return void next(); + } + if (!Core.hasOwners(metadata)) { + cb('E_NO_OWNERS'); + return void next(); + } + + // if you are a pending owner and not an owner + // you can either ADD_OWNERS, or RM_PENDING_OWNERS + // and you should only be able to add yourself as an owner + // everything else should be rejected + // else if you are not an owner + // you should be rejected + // else write the command + + // Confirm that the channel is owned by the user in question + // or the user is accepting a pending ownership offer + if (Core.hasPendingOwners(metadata) && + Core.isPendingOwner(metadata, unsafeKey) && + !Core.isOwner(metadata, unsafeKey)) { + + // If you are a pending owner, make sure you can only add yourelf as an owner + if ((command !== 'ADD_OWNERS' && command !== 'RM_PENDING_OWNERS') + || !Array.isArray(data.value) + || data.value.length !== 1 + || data.value[0] !== unsafeKey) { + cb('INSUFFICIENT_PERMISSIONS'); + return void next(); + } + // FIXME wacky fallthrough is hard to read + // we could pass this off to a writeMetadataCommand function + // and make the flow easier to follow + } else if (!Core.isOwner(metadata, unsafeKey)) { + cb('INSUFFICIENT_PERMISSIONS'); + return void next(); + } + + // Add the new metadata line + var line = [command, data.value, +new Date()]; + var changed = false; + try { + changed = Meta.handleCommand(metadata, line); + } catch (e) { + cb(e); + return void next(); + } + + // if your command is valid but it didn't result in any change to the metadata, + // call back now and don't write any "useless" line to the log + if (!changed) { + cb(void 0, metadata); + return void next(); + } + Env.msgStore.writeMetadata(channel, JSON.stringify(line), function (e) { + if (e) { + cb(e); + return void next(); + } + cb(void 0, metadata); + next(); + }); + }); + }); +}; + + diff --git a/lib/rpc.js b/lib/rpc.js index 3785b20a1..0ddfdb128 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -1,8 +1,5 @@ /*jshint esversion: 6 */ const nThen = require("nthen"); -const Meta = require("./metadata"); -const WriteQueue = require("./write-queue"); -const BatchRead = require("./batch-read"); const Util = require("./common-util"); const escapeKeyCharacters = Util.escapeKeyCharacters; @@ -13,6 +10,7 @@ const Admin = require("./commands/admin-rpc"); const Pinning = require("./commands/pin-rpc"); const Quota = require("./commands/quota"); const Block = require("./commands/block"); +const Metadata = require("./commands/metadata"); var RPC = module.exports; @@ -31,116 +29,12 @@ var WARN = function (e, output) { } }; -const batchMetadata = BatchRead("GET_METADATA"); -var getMetadata = function (Env, channel, cb) { - if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } - if (channel.length !== 32) { return cb("INVALID_CHAN_LENGTH"); } - - batchMetadata(channel, cb, function (done) { - var ref = {}; - var lineHandler = Meta.createLineHandler(ref, Log.error); - - return void Env.msgStore.readChannelMetadata(channel, lineHandler, function (err) { - if (err) { - // stream errors? - return void done(err); - } - done(void 0, ref.meta); - }); - }); -}; - -/* setMetadata - - write a new line to the metadata log if a valid command is provided - - data is an object: { - channel: channelId, - command: metadataCommand (string), - value: value - } -*/ -var queueMetadata = WriteQueue(); -var setMetadata = function (Env, data, unsafeKey, cb) { - var channel = data.channel; - var command = data.command; - if (!channel || !Core.isValidId(channel)) { return void cb ('INVALID_CHAN'); } - if (!command || typeof (command) !== 'string') { return void cb ('INVALID_COMMAND'); } - if (Meta.commands.indexOf(command) === -1) { return void('UNSUPPORTED_COMMAND'); } - - queueMetadata(channel, function (next) { - getMetadata(Env, channel, function (err, metadata) { - if (err) { - cb(err); - return void next(); - } - if (!Core.hasOwners(metadata)) { - cb('E_NO_OWNERS'); - return void next(); - } - - // if you are a pending owner and not an owner - // you can either ADD_OWNERS, or RM_PENDING_OWNERS - // and you should only be able to add yourself as an owner - // everything else should be rejected - // else if you are not an owner - // you should be rejected - // else write the command - - // Confirm that the channel is owned by the user in question - // or the user is accepting a pending ownership offer - if (Core.hasPendingOwners(metadata) && - Core.isPendingOwner(metadata, unsafeKey) && - !Core.isOwner(metadata, unsafeKey)) { - - // If you are a pending owner, make sure you can only add yourelf as an owner - if ((command !== 'ADD_OWNERS' && command !== 'RM_PENDING_OWNERS') - || !Array.isArray(data.value) - || data.value.length !== 1 - || data.value[0] !== unsafeKey) { - cb('INSUFFICIENT_PERMISSIONS'); - return void next(); - } - // FIXME wacky fallthrough is hard to read - // we could pass this off to a writeMetadataCommand function - // and make the flow easier to follow - } else if (!Core.isOwner(metadata, unsafeKey)) { - cb('INSUFFICIENT_PERMISSIONS'); - return void next(); - } - - // Add the new metadata line - var line = [command, data.value, +new Date()]; - var changed = false; - try { - changed = Meta.handleCommand(metadata, line); - } catch (e) { - cb(e); - return void next(); - } - - // if your command is valid but it didn't result in any change to the metadata, - // call back now and don't write any "useless" line to the log - if (!changed) { - cb(void 0, metadata); - return void next(); - } - Env.msgStore.writeMetadata(channel, JSON.stringify(line), function (e) { - if (e) { - cb(e); - return void next(); - } - cb(void 0, metadata); - next(); - }); - }); - }); -}; - var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { if (typeof(channelId) !== 'string' || channelId.length !== 32) { return cb('INVALID_ARGUMENTS'); } - getMetadata(Env, channelId, function (err, metadata) { + Metadata.getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } // Confirm that the channel is owned by the user in question @@ -199,7 +93,7 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { }); } - getMetadata(Env, channelId, function (err, metadata) { + Metadata.getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } if (!Core.isOwner(metadata, unsafeKey)) { @@ -219,7 +113,7 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { nThen(function (w) { - getMetadata(Env, channelId, w(function (err, metadata) { + Metadata.getMetadata(Env, channelId, w(function (err, metadata) { if (err) { return void cb(err); } if (!Core.hasOwners(metadata)) { w.abort(); @@ -463,7 +357,7 @@ RPC.create = function (config, cb) { respond(e, [null, size, null]); }); case 'GET_METADATA': - return void getMetadata(Env, msg[1], function (e, data) { + return void Metadata.getMetadata(Env, msg[1], function (e, data) { WARN(e, msg[1]); respond(e, [null, data, null]); }); @@ -718,7 +612,7 @@ RPC.create = function (config, cb) { Respond(void 0, result); }); case 'SET_METADATA': - return void setMetadata(Env, msg[1], publicKey, function (e, data) { + return void Metadata.setMetadata(Env, msg[1], publicKey, function (e, data) { if (e) { WARN(e, data); return void Respond(e); From ceb351326c1b50146f175a9ea8010333c97acb82 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 13:36:14 -0500 Subject: [PATCH 032/172] split out some more rpc functionality and fix broken module paths --- lib/commands/channel.js | 193 +++++++++++++++++++++++++++++ lib/commands/metadata.js | 4 +- lib/commands/quota.js | 2 +- lib/commands/upload.js | 37 ++++++ lib/rpc.js | 260 ++++----------------------------------- 5 files changed, 255 insertions(+), 241 deletions(-) create mode 100644 lib/commands/channel.js create mode 100644 lib/commands/upload.js diff --git a/lib/commands/channel.js b/lib/commands/channel.js new file mode 100644 index 000000000..052aa3c44 --- /dev/null +++ b/lib/commands/channel.js @@ -0,0 +1,193 @@ +/*jshint esversion: 6 */ +const Channel = module.exports; + +const Util = require("../common-util"); +const nThen = require("nthen"); +const Core = require("./core"); +const Metadata = require("./metadata"); + +Channel.clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { + if (typeof(channelId) !== 'string' || channelId.length !== 32) { + return cb('INVALID_ARGUMENTS'); + } + + Metadata.getMetadata(Env, channelId, function (err, metadata) { + if (err) { return void cb(err); } + if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } + // Confirm that the channel is owned by the user in question + if (!Core.isOwner(metadata, unsafeKey)) { + return void cb('INSUFFICIENT_PERMISSIONS'); + } + return void Env.msgStore.clearChannel(channelId, function (e) { + cb(e); + }); + }); +}; + +Channel.removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { + if (typeof(channelId) !== 'string' || !Core.isValidId(channelId)) { + return cb('INVALID_ARGUMENTS'); + } + + if (Env.blobStore.isFileId(channelId)) { + var safeKey = Util.escapeKeyCharacters(unsafeKey); + var blobId = channelId; + + return void nThen(function (w) { + // check if you have permissions + Env.blobStore.isOwnedBy(safeKey, blobId, w(function (err, owned) { + if (err || !owned) { + w.abort(); + return void cb("INSUFFICIENT_PERMISSIONS"); + } + })); + }).nThen(function (w) { + // remove the blob + return void Env.blobStore.archive.blob(blobId, w(function (err) { + Env.Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', { + safeKey: safeKey, + blobId: blobId, + status: err? String(err): 'SUCCESS', + }); + if (err) { + w.abort(); + return void cb(err); + } + })); + }).nThen(function () { + // archive the proof + return void Env.blobStore.archive.proof(safeKey, blobId, function (err) { + Env.Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", { + safeKey: safeKey, + blobId: blobId, + status: err? String(err): 'SUCCESS', + }); + if (err) { + return void cb("E_PROOF_REMOVAL"); + } + cb(); + }); + }); + } + + Metadata.getMetadata(Env, channelId, function (err, metadata) { + if (err) { return void cb(err); } + if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } + if (!Core.isOwner(metadata, unsafeKey)) { + return void cb('INSUFFICIENT_PERMISSIONS'); + } + // temporarily archive the file + return void Env.msgStore.archiveChannel(channelId, function (e) { + Env.Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', { + unsafeKey: unsafeKey, + channelId: channelId, + status: e? String(e): 'SUCCESS', + }); + cb(e); + }); + }); +}; + +Channel.removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { + nThen(function (w) { + Metadata.getMetadata(Env, channelId, w(function (err, metadata) { + if (err) { return void cb(err); } + if (!Core.hasOwners(metadata)) { + w.abort(); + return void cb('E_NO_OWNERS'); + } + if (!Core.isOwner(metadata, unsafeKey)) { + w.abort(); + return void cb("INSUFFICIENT_PERMISSIONS"); + } + // else fall through to the next block + })); + }).nThen(function () { + Env.msgStore.trimChannel(channelId, hash, function (err) { + if (err) { return void cb(err); } + + + // XXX you must also clear the channel's index from historyKeeper cache + }); + }); +}; + +var ARRAY_LINE = /^\[/; + +/* Files can contain metadata but not content + call back with true if the channel log has no content other than metadata + otherwise false +*/ +Channel.isNewChannel = function (Env, channel, cb) { + if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } + if (channel.length !== 32) { return void cb('INVALID_CHAN'); } + + var done = false; + Env.msgStore.getMessages(channel, function (msg) { + if (done) { return; } + try { + if (typeof(msg) === 'string' && ARRAY_LINE.test(msg)) { + done = true; + return void cb(void 0, false); + } + } catch (e) { + Env.WARN('invalid message read from store', e); + } + }, function () { + if (done) { return; } + // no more messages... + cb(void 0, true); + }); +}; + +/* writePrivateMessage + allows users to anonymously send a message to the channel + prevents their netflux-id from being stored in history + and from being broadcast to anyone that might currently be in the channel + + Otherwise behaves the same as sending to a channel +*/ +Channel.writePrivateMessage = function (Env, args, nfwssCtx, cb) { + var channelId = args[0]; + var msg = args[1]; + + // don't bother handling empty messages + if (!msg) { return void cb("INVALID_MESSAGE"); } + + // don't support anything except regular channels + if (!Core.isValidId(channelId) || channelId.length !== 32) { + return void cb("INVALID_CHAN"); + } + + // We expect a modern netflux-websocket-server instance + // if this API isn't here everything will fall apart anyway + if (!(nfwssCtx && nfwssCtx.historyKeeper && typeof(nfwssCtx.historyKeeper.onChannelMessage) === 'function')) { + return void cb("NOT_IMPLEMENTED"); + } + + // historyKeeper expects something with an 'id' attribute + // it will fail unless you provide it, but it doesn't need anything else + var channelStruct = { + id: channelId, + }; + + // construct a message to store and broadcast + var fullMessage = [ + 0, // idk + null, // normally the netflux id, null isn't rejected, and it distinguishes messages written in this way + "MSG", // indicate that this is a MSG + channelId, // channel id + msg // the actual message content. Generally a string + ]; + + // store the message and do everything else that is typically done when going through historyKeeper + nfwssCtx.historyKeeper.onChannelMessage(nfwssCtx, channelStruct, fullMessage); + + // call back with the message and the target channel. + // historyKeeper will take care of broadcasting it if anyone is in the channel + cb(void 0, { + channel: channelId, + message: fullMessage + }); +}; + diff --git a/lib/commands/metadata.js b/lib/commands/metadata.js index 91b08d8cc..3a21aae0b 100644 --- a/lib/commands/metadata.js +++ b/lib/commands/metadata.js @@ -3,8 +3,8 @@ const Data = module.exports; const Meta = require("../metadata"); const BatchRead = require("../batch-read"); -const WriteQueue = require("./write-queue"); -const Core = require("./commands/core"); +const WriteQueue = require("../write-queue"); +const Core = require("./core"); const batchMetadata = BatchRead("GET_METADATA"); Data.getMetadata = function (Env, channel, cb) { diff --git a/lib/commands/quota.js b/lib/commands/quota.js index e7df14364..92b1c3cd0 100644 --- a/lib/commands/quota.js +++ b/lib/commands/quota.js @@ -2,7 +2,7 @@ /* globals Buffer*/ const Quota = module.exports; -const Core = require("./commands/core"); +const Core = require("./core"); const Util = require("../common-util"); const Package = require('../../package.json'); const Https = require("https"); diff --git a/lib/commands/upload.js b/lib/commands/upload.js new file mode 100644 index 000000000..f89cf2904 --- /dev/null +++ b/lib/commands/upload.js @@ -0,0 +1,37 @@ +/*jshint esversion: 6 */ +const Upload = module.exports; +const Util = require("../common-util"); +const Pinning = require("./pin-rpc"); +const nThen = require("nthen"); + +// upload_status +Upload.upload_status = function (Env, safeKey, filesize, _cb) { // FIXME FILES + var cb = Util.once(Util.mkAsync(_cb)); + + // validate that the provided size is actually a positive number + if (typeof(filesize) !== 'number' && + filesize >= 0) { return void cb('E_INVALID_SIZE'); } + + if (filesize >= Env.maxUploadSize) { return cb('TOO_LARGE'); } + + nThen(function (w) { + var abortAndCB = Util.both(w.abort, cb); + Env.blobStore.status(safeKey, w(function (err, inProgress) { + // if there's an error something is weird + if (err) { return void abortAndCB(err); } + + // we cannot upload two things at once + if (inProgress) { return void abortAndCB(void 0, true); } + })); + }).nThen(function () { + // if yuo're here then there are no pending uploads + // check if you have space in your quota to upload something of this size + Pinning.getFreeSpace(Env, safeKey, function (e, free) { + if (e) { return void cb(e); } + if (filesize >= free) { return cb('NOT_ENOUGH_SPACE'); } + cb(void 0, false); + }); + }); +}; + + diff --git a/lib/rpc.js b/lib/rpc.js index 0ddfdb128..1912868ed 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -2,7 +2,6 @@ const nThen = require("nthen"); const Util = require("./common-util"); -const escapeKeyCharacters = Util.escapeKeyCharacters; const mkEvent = Util.mkEvent; const Core = require("./commands/core"); @@ -11,208 +10,13 @@ const Pinning = require("./commands/pin-rpc"); const Quota = require("./commands/quota"); const Block = require("./commands/block"); const Metadata = require("./commands/metadata"); +const Channel = require("./commands/channel"); +const Upload = require("./commands/upload"); var RPC = module.exports; -var Store = require("../storage/file"); -var BlobStore = require("../storage/blob"); - -var Log; - -var WARN = function (e, output) { - if (e && output) { - Log.warn(e, { - output: output, - message: String(e), - stack: new Error(e).stack, - }); - } -}; - -var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { - if (typeof(channelId) !== 'string' || channelId.length !== 32) { - return cb('INVALID_ARGUMENTS'); - } - - Metadata.getMetadata(Env, channelId, function (err, metadata) { - if (err) { return void cb(err); } - if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } - // Confirm that the channel is owned by the user in question - if (!Core.isOwner(metadata, unsafeKey)) { - return void cb('INSUFFICIENT_PERMISSIONS'); - } - return void Env.msgStore.clearChannel(channelId, function (e) { - cb(e); - }); - }); -}; - -var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { - if (typeof(channelId) !== 'string' || !Core.isValidId(channelId)) { - return cb('INVALID_ARGUMENTS'); - } - - if (Env.blobStore.isFileId(channelId)) { - var safeKey = escapeKeyCharacters(unsafeKey); - var blobId = channelId; - - return void nThen(function (w) { - // check if you have permissions - Env.blobStore.isOwnedBy(safeKey, blobId, w(function (err, owned) { - if (err || !owned) { - w.abort(); - return void cb("INSUFFICIENT_PERMISSIONS"); - } - })); - }).nThen(function (w) { - // remove the blob - return void Env.blobStore.archive.blob(blobId, w(function (err) { - Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', { - safeKey: safeKey, - blobId: blobId, - status: err? String(err): 'SUCCESS', - }); - if (err) { - w.abort(); - return void cb(err); - } - })); - }).nThen(function () { - // archive the proof - return void Env.blobStore.archive.proof(safeKey, blobId, function (err) { - Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", { - safeKey: safeKey, - blobId: blobId, - status: err? String(err): 'SUCCESS', - }); - if (err) { - return void cb("E_PROOF_REMOVAL"); - } - cb(); - }); - }); - } - - Metadata.getMetadata(Env, channelId, function (err, metadata) { - if (err) { return void cb(err); } - if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } - if (!Core.isOwner(metadata, unsafeKey)) { - return void cb('INSUFFICIENT_PERMISSIONS'); - } - // temporarily archive the file - return void Env.msgStore.archiveChannel(channelId, function (e) { - Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', { - unsafeKey: unsafeKey, - channelId: channelId, - status: e? String(e): 'SUCCESS', - }); - cb(e); - }); - }); -}; - -var removeOwnedChannelHistory = function (Env, channelId, unsafeKey, hash, cb) { - nThen(function (w) { - Metadata.getMetadata(Env, channelId, w(function (err, metadata) { - if (err) { return void cb(err); } - if (!Core.hasOwners(metadata)) { - w.abort(); - return void cb('E_NO_OWNERS'); - } - if (!Core.isOwner(metadata, unsafeKey)) { - w.abort(); - return void cb("INSUFFICIENT_PERMISSIONS"); - } - // else fall through to the next block - })); - }).nThen(function () { - Env.msgStore.trimChannel(channelId, hash, function (err) { - if (err) { return void cb(err); } - - - // XXX you must also clear the channel's index from historyKeeper cache - }); - }); -}; - -var ARRAY_LINE = /^\[/; - -/* Files can contain metadata but not content - call back with true if the channel log has no content other than metadata - otherwise false -*/ -var isNewChannel = function (Env, channel, cb) { - if (!Core.isValidId(channel)) { return void cb('INVALID_CHAN'); } - if (channel.length !== 32) { return void cb('INVALID_CHAN'); } - - var done = false; - Env.msgStore.getMessages(channel, function (msg) { - if (done) { return; } - try { - if (typeof(msg) === 'string' && ARRAY_LINE.test(msg)) { - done = true; - return void cb(void 0, false); - } - } catch (e) { - WARN('invalid message read from store', e); - } - }, function () { - if (done) { return; } - // no more messages... - cb(void 0, true); - }); -}; - -/* writePrivateMessage - allows users to anonymously send a message to the channel - prevents their netflux-id from being stored in history - and from being broadcast to anyone that might currently be in the channel - - Otherwise behaves the same as sending to a channel -*/ -var writePrivateMessage = function (Env, args, nfwssCtx, cb) { - var channelId = args[0]; - var msg = args[1]; - - // don't bother handling empty messages - if (!msg) { return void cb("INVALID_MESSAGE"); } - - // don't support anything except regular channels - if (!Core.isValidId(channelId) || channelId.length !== 32) { - return void cb("INVALID_CHAN"); - } - - // We expect a modern netflux-websocket-server instance - // if this API isn't here everything will fall apart anyway - if (!(nfwssCtx && nfwssCtx.historyKeeper && typeof(nfwssCtx.historyKeeper.onChannelMessage) === 'function')) { - return void cb("NOT_IMPLEMENTED"); - } - - // historyKeeper expects something with an 'id' attribute - // it will fail unless you provide it, but it doesn't need anything else - var channelStruct = { - id: channelId, - }; - - // construct a message to store and broadcast - var fullMessage = [ - 0, // idk - null, // normally the netflux id, null isn't rejected, and it distinguishes messages written in this way - "MSG", // indicate that this is a MSG - channelId, // channel id - msg // the actual message content. Generally a string - ]; - - // store the message and do everything else that is typically done when going through historyKeeper - nfwssCtx.historyKeeper.onChannelMessage(nfwssCtx, channelStruct, fullMessage); - - // call back with the message and the target channel. - // historyKeeper will take care of broadcasting it if anyone is in the channel - cb(void 0, { - channel: channelId, - message: fullMessage - }); -}; +const Store = require("../storage/file"); +const BlobStore = require("../storage/blob"); var isUnauthenticatedCall = function (call) { return [ @@ -254,38 +58,8 @@ var isAuthenticatedCall = function (call) { ].indexOf(call) !== -1; }; -// upload_status -var upload_status = function (Env, safeKey, filesize, _cb) { // FIXME FILES - var cb = Util.once(Util.mkAsync(_cb)); - - // validate that the provided size is actually a positive number - if (typeof(filesize) !== 'number' && - filesize >= 0) { return void cb('E_INVALID_SIZE'); } - - if (filesize >= Env.maxUploadSize) { return cb('TOO_LARGE'); } - - nThen(function (w) { - var abortAndCB = Util.both(w.abort, cb); - Env.blobStore.status(safeKey, w(function (err, inProgress) { - // if there's an error something is weird - if (err) { return void abortAndCB(err); } - - // we cannot upload two things at once - if (inProgress) { return void abortAndCB(void 0, true); } - })); - }).nThen(function () { - // if yuo're here then there are no pending uploads - // check if you have space in your quota to upload something of this size - Pinning.getFreeSpace(Env, safeKey, function (e, free) { - if (e) { return void cb(e); } - if (filesize >= free) { return cb('NOT_ENOUGH_SPACE'); } - cb(void 0, false); - }); - }); -}; - RPC.create = function (config, cb) { - Log = config.log; + var Log = config.log; // load pin-store... Log.silly('LOADING RPC MODULE'); @@ -294,6 +68,16 @@ RPC.create = function (config, cb) { return typeof(config[key]) === 'string'? config[key]: def; }; + var WARN = function (e, output) { + if (e && output) { + Log.warn(e, { + output: output, + message: String(e), + stack: new Error(e).stack, + }); + } + }; + var Env = { defaultStorageLimit: config.defaultStorageLimit, maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024), @@ -382,11 +166,11 @@ RPC.create = function (config, cb) { respond(null, [null, isPinned, null]); }); case 'IS_NEW_CHANNEL': - return void isNewChannel(Env, msg[1], function (e, isNew) { + return void Channel.isNewChannel(Env, msg[1], function (e, isNew) { respond(e, [null, isNew, null]); }); case 'WRITE_PRIVATE_MESSAGE': - return void writePrivateMessage(Env, msg[1], nfwssCtx, function (e, output) { + return void Channel.writePrivateMessage(Env, msg[1], nfwssCtx, function (e, output) { respond(e, output); }); default: @@ -450,7 +234,7 @@ RPC.create = function (config, cb) { return void respond("INVALID_RPC_CALL"); } - var safeKey = escapeKeyCharacters(publicKey); + var safeKey = Util.escapeKeyCharacters(publicKey); /* If you have gotten this far, you have signed the message with the public key which you provided. @@ -528,18 +312,18 @@ RPC.create = function (config, cb) { Respond(void 0, "OK"); }); case 'CLEAR_OWNED_CHANNEL': - return void clearOwnedChannel(Env, msg[1], publicKey, function (e, response) { + return void Channel.clearOwnedChannel(Env, msg[1], publicKey, function (e, response) { if (e) { return void Respond(e); } Respond(void 0, response); }); case 'REMOVE_OWNED_CHANNEL': - return void removeOwnedChannel(Env, msg[1], publicKey, function (e) { + return void Channel.removeOwnedChannel(Env, msg[1], publicKey, function (e) { if (e) { return void Respond(e); } Respond(void 0, "OK"); }); case 'TRIM_OWNED_CHANNEL_HISTORY': - return void removeOwnedChannelHistory(Env, msg[1], publicKey, msg[2], function (e) { + return void Channel.removeOwnedChannelHistory(Env, msg[1], publicKey, msg[2], function (e) { if (e) { return void Respond(e); } Respond(void 0, 'OK'); }); @@ -560,7 +344,7 @@ RPC.create = function (config, cb) { }); case 'UPLOAD_STATUS': var filesize = msg[1]; - return void upload_status(Env, safeKey, filesize, function (e, yes) { + return void Upload.upload_status(Env, safeKey, filesize, function (e, yes) { if (!e && !yes) { // no pending uploads, set the new size var user = Core.getSession(Sessions, safeKey); From b093d3f0d2cb75472e15235e0cbd4589b8d5cbff Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 24 Jan 2020 14:45:53 -0500 Subject: [PATCH 033/172] WIP massive rpc refactor --- lib/commands/admin-rpc.js | 4 +- lib/commands/quota.js | 31 +- lib/rpc.js | 713 ++++++++++++++++++++------------------ 3 files changed, 386 insertions(+), 362 deletions(-) diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 9e32fe2a8..24a8e4348 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -93,7 +93,7 @@ var getDiskUsage = function (Env, cb) { -Admin.command = function (Env, ctx, publicKey, config, data, cb) { +Admin.command = function (Env, ctx, publicKey, data, cb) { var admins = Env.admins; if (admins.indexOf(publicKey) === -1) { return void cb("FORBIDDEN"); @@ -109,7 +109,7 @@ Admin.command = function (Env, ctx, publicKey, config, data, cb) { case 'DISK_USAGE': return getDiskUsage(Env, cb); case 'FLUSH_CACHE': - config.flushCache(); + Env.flushCache(); return cb(void 0, true); case 'SHUTDOWN': return shutdown(Env, ctx, cb); diff --git a/lib/commands/quota.js b/lib/commands/quota.js index 92b1c3cd0..b74195821 100644 --- a/lib/commands/quota.js +++ b/lib/commands/quota.js @@ -7,7 +7,7 @@ const Util = require("../common-util"); const Package = require('../../package.json'); const Https = require("https"); -Quota.applyCustomLimits = function (Env, config) { +Quota.applyCustomLimits = function (Env) { var isLimit = function (o) { var valid = o && typeof(o) === 'object' && typeof(o.limit) === 'number' && @@ -16,7 +16,7 @@ Quota.applyCustomLimits = function (Env, config) { return valid; }; - // read custom limits from the config + // read custom limits from the Environment (taken from config) var customLimits = (function (custom) { var limits = {}; Object.keys(custom).forEach(function (k) { @@ -27,7 +27,7 @@ Quota.applyCustomLimits = function (Env, config) { }); }); return limits; - }(config.customLimits || {})); + }(Env.customLimits || {})); Object.keys(customLimits).forEach(function (k) { if (!isLimit(customLimits[k])) { return; } @@ -37,17 +37,18 @@ Quota.applyCustomLimits = function (Env, config) { // The limits object contains storage limits for all the publicKey that have paid // To each key is associated an object containing the 'limit' value and a 'note' explaining that limit -Quota.updateLimits = function (Env, config, publicKey, cb) { // FIXME BATCH?S +// XXX maybe the use case with a publicKey should be a different command that calls this? +Quota.updateLimits = function (Env, publicKey, cb) { // FIXME BATCH?S - if (config.adminEmail === false) { - Quota.applyCustomLimits(Env, config); - if (config.allowSubscriptions === false) { return; } + if (Env.adminEmail === false) { + Quota.applyCustomLimits(Env); + if (Env.allowSubscriptions === false) { return; } throw new Error("allowSubscriptions must be false if adminEmail is false"); } if (typeof cb !== "function") { cb = function () {}; } - var defaultLimit = typeof(config.defaultStorageLimit) === 'number'? - config.defaultStorageLimit: Core.DEFAULT_LIMIT; + var defaultLimit = typeof(Env.defaultStorageLimit) === 'number'? + Env.defaultStorageLimit: Core.DEFAULT_LIMIT; var userId; if (publicKey) { @@ -55,9 +56,9 @@ Quota.updateLimits = function (Env, config, publicKey, cb) { // FIXME BATCH?S } var body = JSON.stringify({ - domain: config.myDomain, - subdomain: config.mySubdomain || null, - adminEmail: config.adminEmail, + domain: Env.myDomain, + subdomain: Env.mySubdomain || null, + adminEmail: Env.adminEmail, version: Package.version }); var options = { @@ -84,7 +85,7 @@ Quota.updateLimits = function (Env, config, publicKey, cb) { // FIXME BATCH?S try { var json = JSON.parse(str); Env.limits = json; - Quota.applyCustomLimits(Env, config); + Quota.applyCustomLimits(Env); var l; if (userId) { @@ -100,8 +101,8 @@ Quota.updateLimits = function (Env, config, publicKey, cb) { // FIXME BATCH?S }); req.on('error', function (e) { - Quota.applyCustomLimits(Env, config); - if (!config.domain) { return cb(); } + Quota.applyCustomLimits(Env); + if (!Env.domain) { return cb(); } // XXX cb(e); }); diff --git a/lib/rpc.js b/lib/rpc.js index 1912868ed..171fb590f 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -18,44 +18,359 @@ var RPC = module.exports; const Store = require("../storage/file"); const BlobStore = require("../storage/blob"); +const UNAUTHENTICATED_CALLS = [ + 'GET_FILE_SIZE', + 'GET_METADATA', + 'GET_MULTIPLE_FILE_SIZE', + 'IS_CHANNEL_PINNED', + 'IS_NEW_CHANNEL', + 'GET_HISTORY_OFFSET', + 'GET_DELETED_PADS', + 'WRITE_PRIVATE_MESSAGE', +]; + var isUnauthenticatedCall = function (call) { - return [ - 'GET_FILE_SIZE', - 'GET_METADATA', - 'GET_MULTIPLE_FILE_SIZE', - 'IS_CHANNEL_PINNED', - 'IS_NEW_CHANNEL', - 'GET_HISTORY_OFFSET', - 'GET_DELETED_PADS', - 'WRITE_PRIVATE_MESSAGE', - ].indexOf(call) !== -1; + return UNAUTHENTICATED_CALLS.indexOf(call) !== -1; }; +const AUTHENTICATED_CALLS = [ + 'COOKIE', + 'RESET', + 'PIN', + 'UNPIN', + 'GET_HASH', + 'GET_TOTAL_SIZE', + 'UPDATE_LIMITS', + 'GET_LIMIT', + 'UPLOAD_STATUS', + 'UPLOAD_COMPLETE', + 'OWNED_UPLOAD_COMPLETE', + 'UPLOAD_CANCEL', + 'EXPIRE_SESSION', + 'TRIM_OWNED_CHANNEL_HISTORY', + 'CLEAR_OWNED_CHANNEL', + 'REMOVE_OWNED_CHANNEL', + 'REMOVE_PINS', + 'TRIM_PINS', + 'WRITE_LOGIN_BLOCK', + 'REMOVE_LOGIN_BLOCK', + 'ADMIN', + 'SET_METADATA' +]; + var isAuthenticatedCall = function (call) { - return [ - 'COOKIE', - 'RESET', - 'PIN', - 'UNPIN', - 'GET_HASH', - 'GET_TOTAL_SIZE', - 'UPDATE_LIMITS', - 'GET_LIMIT', - 'UPLOAD_STATUS', - 'UPLOAD_COMPLETE', - 'OWNED_UPLOAD_COMPLETE', - 'UPLOAD_CANCEL', - 'EXPIRE_SESSION', - 'TRIM_OWNED_CHANNEL_HISTORY', - 'CLEAR_OWNED_CHANNEL', - 'REMOVE_OWNED_CHANNEL', - 'REMOVE_PINS', - 'TRIM_PINS', - 'WRITE_LOGIN_BLOCK', - 'REMOVE_LOGIN_BLOCK', - 'ADMIN', - 'SET_METADATA' - ].indexOf(call) !== -1; + return AUTHENTICATED_CALLS.indexOf(call) !== -1; +}; + +var isUnauthenticateMessage = function (msg) { + return msg && msg.length === 2 && isUnauthenticatedCall(msg[0]); +}; + +var handleUnauthenticatedMessage = function (Env, msg, respond, nfwssCtx) { + Env.Log.silly('LOG_RPC', msg[0]); + switch (msg[0]) { + case 'GET_HISTORY_OFFSET': { // XXX not actually used anywhere? + if (typeof(msg[1]) !== 'object' || typeof(msg[1].channelName) !== 'string') { + return respond('INVALID_ARG_FORMAT', msg); + } + const msgHash = typeof(msg[1].msgHash) === 'string' ? msg[1].msgHash : undefined; + nfwssCtx.getHistoryOffset(nfwssCtx, msg[1].channelName, msgHash, (e, ret) => { + if (e) { + if (e.code !== 'ENOENT') { + Env.WARN(e.stack, msg); + } + return respond(e.message); + } + respond(e, [null, ret, null]); + }); + break; + } + case 'GET_FILE_SIZE': + return void Pinning.getFileSize(Env, msg[1], function (e, size) { + Env.WARN(e, msg[1]); + respond(e, [null, size, null]); + }); + case 'GET_METADATA': + return void Metadata.getMetadata(Env, msg[1], function (e, data) { + Env.WARN(e, msg[1]); + respond(e, [null, data, null]); + }); + case 'GET_MULTIPLE_FILE_SIZE': // XXX not actually used on the client? + return void Pinning.getMultipleFileSize(Env, msg[1], function (e, dict) { + if (e) { + Env.WARN(e, dict); + return respond(e); + } + respond(e, [null, dict, null]); + }); + case 'GET_DELETED_PADS': + return void Pinning.getDeletedPads(Env, msg[1], function (e, list) { + if (e) { + Env.WARN(e, msg[1]); + return respond(e); + } + respond(e, [null, list, null]); + }); + case 'IS_CHANNEL_PINNED': + return void Pinning.isChannelPinned(Env, msg[1], function (isPinned) { + respond(null, [null, isPinned, null]); + }); + case 'IS_NEW_CHANNEL': + return void Channel.isNewChannel(Env, msg[1], function (e, isNew) { + respond(e, [null, isNew, null]); + }); + case 'WRITE_PRIVATE_MESSAGE': + return void Channel.writePrivateMessage(Env, msg[1], nfwssCtx, function (e, output) { + respond(e, output); + }); + default: + Env.Log.warn("UNSUPPORTED_RPC_CALL", msg); + return respond('UNSUPPORTED_RPC_CALL', msg); + } +}; + +var handleAuthenticatedMessage = function (Env, map) { + var msg = map.msg; + var safeKey = map.safeKey; + var publicKey = map.publicKey; + var Respond = map.Respond; + var ctx = map.ctx; + + Env.Log.silly('LOG_RPC', msg[0]); + switch (msg[0]) { + case 'COOKIE': return void Respond(void 0); + case 'RESET': + return Pinning.resetUserPins(Env, safeKey, msg[1], function (e, hash) { // XXX USER_TARGETED + //WARN(e, hash); + return void Respond(e, hash); + }); + case 'PIN': + return Pinning.pinChannel(Env, safeKey, msg[1], function (e, hash) { // XXX USER_TARGETED + Env.WARN(e, hash); + Respond(e, hash); + }); + case 'UNPIN': + return Pinning.unpinChannel(Env, safeKey, msg[1], function (e, hash) { // XXX USER_TARGETED + Env.WARN(e, hash); + Respond(e, hash); + }); + case 'GET_HASH': + return void Pinning.getHash(Env, safeKey, function (e, hash) { // XXX USER_SCOPED + Env.WARN(e, hash); + Respond(e, hash); + }); + case 'GET_TOTAL_SIZE': // TODO cache this, since it will get called quite a bit + return Pinning.getTotalSize(Env, safeKey, function (e, size) { // XXX USER_SCOPED + if (e) { + Env.WARN(e, safeKey); + return void Respond(e); + } + Respond(e, size); + }); + case 'UPDATE_LIMITS': + return void Quota.updateLimits(Env, safeKey, function (e, limit) { // XXX USER_SCOPED + if (e) { + Env.WARN(e, limit); + return void Respond(e); + } + Respond(void 0, limit); + }); + case 'GET_LIMIT': + return void Pinning.getLimit(Env, safeKey, function (e, limit) { // XXX USER_SCOPED + if (e) { + Env.WARN(e, limit); + return void Respond(e); + } + Respond(void 0, limit); + }); + case 'EXPIRE_SESSION': + return void setTimeout(function () { // XXX USER_SCOPED + Core.expireSession(Env.Sessions, safeKey); + Respond(void 0, "OK"); + }); + case 'CLEAR_OWNED_CHANNEL': + return void Channel.clearOwnedChannel(Env, msg[1], publicKey, function (e, response) { // XXX USER_TARGETD_INVERSE + if (e) { return void Respond(e); } + Respond(void 0, response); + }); + + case 'REMOVE_OWNED_CHANNEL': + return void Channel.removeOwnedChannel(Env, msg[1], publicKey, function (e) { // XXX USER_TARGETED_INVERSE + if (e) { return void Respond(e); } + Respond(void 0, "OK"); + }); + case 'TRIM_OWNED_CHANNEL_HISTORY': + return void Channel.removeOwnedChannelHistory(Env, msg[1], publicKey, msg[2], function (e) { // XXX USER_TARGETED_DOUBLE + if (e) { return void Respond(e); } + Respond(void 0, 'OK'); + }); + case 'REMOVE_PINS': + return void Pinning.removePins(Env, safeKey, function (e) { // XXX USER_SCOPED + if (e) { return void Respond(e); } + Respond(void 0, "OK"); + }); + case 'TRIM_PINS': + return void Pinning.trimPins(Env, safeKey, function (e) { // XXX USER_SCOPED + if (e) { return void Respond(e); } + Respond(void 0, "OK"); + }); + case 'UPLOAD': + return void Env.blobStore.upload(safeKey, msg[1], function (e, len) { // XXX USER_SCOPED_SPECIAL + Env.WARN(e, len); + Respond(e, len); + }); + case 'UPLOAD_STATUS': + var filesize = msg[1]; + return void Upload.upload_status(Env, safeKey, filesize, function (e, yes) { // XXX USER_TARGETED + if (!e && !yes) { + // no pending uploads, set the new size + var user = Core.getSession(Env.Sessions, safeKey); + user.pendingUploadSize = filesize; + user.currentUploadSize = 0; + } + Respond(e, yes); + }); + case 'UPLOAD_COMPLETE': + return void Env.blobStore.complete(safeKey, msg[1], function (e, hash) { // XXX USER_SCOPED_SPECIAL + Env.WARN(e, hash); + Respond(e, hash); + }); + case 'OWNED_UPLOAD_COMPLETE': + return void Env.blobStore.completeOwned(safeKey, msg[1], function (e, blobId) { // XXX USER_SCOPED_SPECIAL + Env.WARN(e, blobId); + Respond(e, blobId); + }); + case 'UPLOAD_CANCEL': + // msg[1] is fileSize + // if we pass it here, we can start an upload right away without calling + // UPLOAD_STATUS again + return void Env.blobStore.cancel(safeKey, msg[1], function (e) { // XXX USER_SCOPED_SPECIAL + Env.WARN(e, 'UPLOAD_CANCEL'); + Respond(e); + }); + case 'WRITE_LOGIN_BLOCK': + return void Block.writeLoginBlock(Env, msg[1], function (e) { // XXX SPECIAL + if (e) { + Env.WARN(e, 'WRITE_LOGIN_BLOCK'); + return void Respond(e); + } + Respond(e); + }); + case 'REMOVE_LOGIN_BLOCK': + return void Block.removeLoginBlock(Env, msg[1], function (e) { // XXX SPECIAL + if (e) { + Env.WARN(e, 'REMOVE_LOGIN_BLOCK'); + return void Respond(e); + } + Respond(e); + }); + case 'ADMIN': + return void Admin.command(Env, ctx, safeKey, msg[1], function (e, result) { // XXX SPECIAL + if (e) { + Env.WARN(e, result); + return void Respond(e); + } + Respond(void 0, result); + }); + case 'SET_METADATA': + return void Metadata.setMetadata(Env, msg[1], publicKey, function (e, data) { // XXX USER_TARGETED_INVERSE + if (e) { + Env.WARN(e, data); + return void Respond(e); + } + Respond(void 0, data); + }); + default: + return void Respond('UNSUPPORTED_RPC_CALL', msg); + } +}; + +var rpc = function (Env, ctx, data, respond) { + if (!Array.isArray(data)) { + Env.Log.debug('INVALID_ARG_FORMET', data); + return void respond('INVALID_ARG_FORMAT'); + } + + if (!data.length) { + return void respond("INSUFFICIENT_ARGS"); + } else if (data.length !== 1) { + Env.Log.debug('UNEXPECTED_ARGUMENTS_LENGTH', data); + } + + var msg = data[0].slice(0); + + if (!Array.isArray(msg)) { + return void respond('INVALID_ARG_FORMAT'); + } + + if (isUnauthenticateMessage(msg)) { + return handleUnauthenticatedMessage(Env, msg, respond, ctx); + } + + var signature = msg.shift(); + var publicKey = msg.shift(); + + // make sure a user object is initialized in the cookie jar + if (publicKey) { + Core.getSession(Env.Sessions, publicKey); + } else { + Env.Log.debug("NO_PUBLIC_KEY_PROVIDED", publicKey); + } + + var cookie = msg[0]; + if (!Core.isValidCookie(Env.Sessions, publicKey, cookie)) { + // no cookie is fine if the RPC is to get a cookie + if (msg[1] !== 'COOKIE') { + return void respond('NO_COOKIE'); + } + } + + var serialized = JSON.stringify(msg); + + if (!(serialized && typeof(publicKey) === 'string')) { + return void respond('INVALID_MESSAGE_OR_PUBLIC_KEY'); + } + + if (isAuthenticatedCall(msg[1])) { + if (Core.checkSignature(Env, serialized, signature, publicKey) !== true) { + return void respond("INVALID_SIGNATURE_OR_PUBLIC_KEY"); + } + } else if (msg[1] !== 'UPLOAD') { + Env.Log.warn('INVALID_RPC_CALL', msg[1]); + return void respond("INVALID_RPC_CALL"); + } + + var safeKey = Util.escapeKeyCharacters(publicKey); + /* If you have gotten this far, you have signed the message with the + public key which you provided. + + We can safely modify the state for that key + + OR it's an unauthenticated call, which must not modify the state + for that key in a meaningful way. + */ + + // discard validated cookie from message + msg.shift(); + + var Respond = function (e, msg) { + var session = Env.Sessions[safeKey]; + var token = session? session.tokens.slice(-1)[0]: ''; + var cookie = Core.makeCookie(token).join('|'); + respond(e ? String(e): e, [cookie].concat(typeof(msg) !== 'undefined' ?msg: [])); + }; + + if (typeof(msg) !== 'object' || !msg.length) { + return void Respond('INVALID_MSG'); + } + + handleAuthenticatedMessage(Env, { + msg: msg, + safeKey: safeKey, + publicKey: publicKey, + Respond: Respond, + ctx: ctx, + }); }; RPC.create = function (config, cb) { @@ -92,6 +407,13 @@ RPC.create = function (config, cb) { sessionExpirationInterval: undefined, Log: Log, WARN: WARN, + flushCache: config.flushCache, + adminEmail: config.adminEmail, + allowSubscriptions: config.allowSubscriptions, + myDomain: config.myDomain, + mySubdomain: config.mySubdomain, + customLimits: config.customLimits, + domain: config.domain // XXX }; try { @@ -112,322 +434,14 @@ RPC.create = function (config, cb) { paths.staging = keyOrDefaultString('blobStagingPath', './blobstage'); paths.blob = keyOrDefaultString('blobPath', './blob'); - var isUnauthenticateMessage = function (msg) { - return msg && msg.length === 2 && isUnauthenticatedCall(msg[0]); - }; - - var handleUnauthenticatedMessage = function (msg, respond, nfwssCtx) { - Log.silly('LOG_RPC', msg[0]); - switch (msg[0]) { - case 'GET_HISTORY_OFFSET': { - if (typeof(msg[1]) !== 'object' || typeof(msg[1].channelName) !== 'string') { - return respond('INVALID_ARG_FORMAT', msg); - } - const msgHash = typeof(msg[1].msgHash) === 'string' ? msg[1].msgHash : undefined; - nfwssCtx.getHistoryOffset(nfwssCtx, msg[1].channelName, msgHash, (e, ret) => { - if (e) { - if (e.code !== 'ENOENT') { - WARN(e.stack, msg); - } - return respond(e.message); - } - respond(e, [null, ret, null]); - }); - break; - } - case 'GET_FILE_SIZE': - return void Pinning.getFileSize(Env, msg[1], function (e, size) { - WARN(e, msg[1]); - respond(e, [null, size, null]); - }); - case 'GET_METADATA': - return void Metadata.getMetadata(Env, msg[1], function (e, data) { - WARN(e, msg[1]); - respond(e, [null, data, null]); - }); - case 'GET_MULTIPLE_FILE_SIZE': - return void Pinning.getMultipleFileSize(Env, msg[1], function (e, dict) { - if (e) { - WARN(e, dict); - return respond(e); - } - respond(e, [null, dict, null]); - }); - case 'GET_DELETED_PADS': - return void Pinning.getDeletedPads(Env, msg[1], function (e, list) { - if (e) { - WARN(e, msg[1]); - return respond(e); - } - respond(e, [null, list, null]); - }); - case 'IS_CHANNEL_PINNED': - return void Pinning.isChannelPinned(Env, msg[1], function (isPinned) { - respond(null, [null, isPinned, null]); - }); - case 'IS_NEW_CHANNEL': - return void Channel.isNewChannel(Env, msg[1], function (e, isNew) { - respond(e, [null, isNew, null]); - }); - case 'WRITE_PRIVATE_MESSAGE': - return void Channel.writePrivateMessage(Env, msg[1], nfwssCtx, function (e, output) { - respond(e, output); - }); - default: - Log.warn("UNSUPPORTED_RPC_CALL", msg); - return respond('UNSUPPORTED_RPC_CALL', msg); - } - }; - - var rpc0 = function (ctx, data, respond) { - if (!Array.isArray(data)) { - Log.debug('INVALID_ARG_FORMET', data); - return void respond('INVALID_ARG_FORMAT'); - } - - if (!data.length) { - return void respond("INSUFFICIENT_ARGS"); - } else if (data.length !== 1) { - Log.debug('UNEXPECTED_ARGUMENTS_LENGTH', data); - } - - var msg = data[0].slice(0); - - if (!Array.isArray(msg)) { - return void respond('INVALID_ARG_FORMAT'); - } - - if (isUnauthenticateMessage(msg)) { - return handleUnauthenticatedMessage(msg, respond, ctx); - } - - var signature = msg.shift(); - var publicKey = msg.shift(); - - // make sure a user object is initialized in the cookie jar - if (publicKey) { - Core.getSession(Sessions, publicKey); - } else { - Log.debug("NO_PUBLIC_KEY_PROVIDED", publicKey); - } - - var cookie = msg[0]; - if (!Core.isValidCookie(Sessions, publicKey, cookie)) { - // no cookie is fine if the RPC is to get a cookie - if (msg[1] !== 'COOKIE') { - return void respond('NO_COOKIE'); - } - } - - var serialized = JSON.stringify(msg); - - if (!(serialized && typeof(publicKey) === 'string')) { - return void respond('INVALID_MESSAGE_OR_PUBLIC_KEY'); - } - - if (isAuthenticatedCall(msg[1])) { - if (Core.checkSignature(Env, serialized, signature, publicKey) !== true) { - return void respond("INVALID_SIGNATURE_OR_PUBLIC_KEY"); - } - } else if (msg[1] !== 'UPLOAD') { - Log.warn('INVALID_RPC_CALL', msg[1]); - return void respond("INVALID_RPC_CALL"); - } - - var safeKey = Util.escapeKeyCharacters(publicKey); - /* If you have gotten this far, you have signed the message with the - public key which you provided. - - We can safely modify the state for that key - - OR it's an unauthenticated call, which must not modify the state - for that key in a meaningful way. - */ - - // discard validated cookie from message - msg.shift(); - - var Respond = function (e, msg) { - var session = Sessions[safeKey]; - var token = session? session.tokens.slice(-1)[0]: ''; - var cookie = Core.makeCookie(token).join('|'); - respond(e ? String(e): e, [cookie].concat(typeof(msg) !== 'undefined' ?msg: [])); - }; - - if (typeof(msg) !== 'object' || !msg.length) { - return void Respond('INVALID_MSG'); - } - - var handleMessage = function () { - Log.silly('LOG_RPC', msg[0]); - switch (msg[0]) { - case 'COOKIE': return void Respond(void 0); - case 'RESET': - return Pinning.resetUserPins(Env, safeKey, msg[1], function (e, hash) { - //WARN(e, hash); - return void Respond(e, hash); - }); - case 'PIN': - return Pinning.pinChannel(Env, safeKey, msg[1], function (e, hash) { - WARN(e, hash); - Respond(e, hash); - }); - case 'UNPIN': - return Pinning.unpinChannel(Env, safeKey, msg[1], function (e, hash) { - WARN(e, hash); - Respond(e, hash); - }); - case 'GET_HASH': - return void Pinning.getHash(Env, safeKey, function (e, hash) { - WARN(e, hash); - Respond(e, hash); - }); - case 'GET_TOTAL_SIZE': // TODO cache this, since it will get called quite a bit - return Pinning.getTotalSize(Env, safeKey, function (e, size) { - if (e) { - WARN(e, safeKey); - return void Respond(e); - } - Respond(e, size); - }); - case 'UPDATE_LIMITS': - return void Quota.updateLimits(Env, config, safeKey, function (e, limit) { - if (e) { - WARN(e, limit); - return void Respond(e); - } - Respond(void 0, limit); - }); - case 'GET_LIMIT': - return void Pinning.getLimit(Env, safeKey, function (e, limit) { - if (e) { - WARN(e, limit); - return void Respond(e); - } - Respond(void 0, limit); - }); - case 'EXPIRE_SESSION': - return void setTimeout(function () { - Core.expireSession(Sessions, safeKey); - Respond(void 0, "OK"); - }); - case 'CLEAR_OWNED_CHANNEL': - return void Channel.clearOwnedChannel(Env, msg[1], publicKey, function (e, response) { - if (e) { return void Respond(e); } - Respond(void 0, response); - }); - - case 'REMOVE_OWNED_CHANNEL': - return void Channel.removeOwnedChannel(Env, msg[1], publicKey, function (e) { - if (e) { return void Respond(e); } - Respond(void 0, "OK"); - }); - case 'TRIM_OWNED_CHANNEL_HISTORY': - return void Channel.removeOwnedChannelHistory(Env, msg[1], publicKey, msg[2], function (e) { - if (e) { return void Respond(e); } - Respond(void 0, 'OK'); - }); - case 'REMOVE_PINS': - return void Pinning.removePins(Env, safeKey, function (e) { - if (e) { return void Respond(e); } - Respond(void 0, "OK"); - }); - case 'TRIM_PINS': - return void Pinning.trimPins(Env, safeKey, function (e) { - if (e) { return void Respond(e); } - Respond(void 0, "OK"); - }); - case 'UPLOAD': - return void Env.blobStore.upload(safeKey, msg[1], function (e, len) { - WARN(e, len); - Respond(e, len); - }); - case 'UPLOAD_STATUS': - var filesize = msg[1]; - return void Upload.upload_status(Env, safeKey, filesize, function (e, yes) { - if (!e && !yes) { - // no pending uploads, set the new size - var user = Core.getSession(Sessions, safeKey); - user.pendingUploadSize = filesize; - user.currentUploadSize = 0; - } - Respond(e, yes); - }); - case 'UPLOAD_COMPLETE': - return void Env.blobStore.complete(safeKey, msg[1], function (e, hash) { - WARN(e, hash); - Respond(e, hash); - }); - case 'OWNED_UPLOAD_COMPLETE': - return void Env.blobStore.completeOwned(safeKey, msg[1], function (e, blobId) { - WARN(e, blobId); - Respond(e, blobId); - }); - case 'UPLOAD_CANCEL': - // msg[1] is fileSize - // if we pass it here, we can start an upload right away without calling - // UPLOAD_STATUS again - return void Env.blobStore.cancel(safeKey, msg[1], function (e) { - WARN(e, 'UPLOAD_CANCEL'); - Respond(e); - }); - case 'WRITE_LOGIN_BLOCK': - return void Block.writeLoginBlock(Env, msg[1], function (e) { - if (e) { - WARN(e, 'WRITE_LOGIN_BLOCK'); - return void Respond(e); - } - Respond(e); - }); - case 'REMOVE_LOGIN_BLOCK': - return void Block.removeLoginBlock(Env, msg[1], function (e) { - if (e) { - WARN(e, 'REMOVE_LOGIN_BLOCK'); - return void Respond(e); - } - Respond(e); - }); - case 'ADMIN': - return void Admin.command(Env, ctx, safeKey, config, msg[1], function (e, result) { - if (e) { - WARN(e, result); - return void Respond(e); - } - Respond(void 0, result); - }); - case 'SET_METADATA': - return void Metadata.setMetadata(Env, msg[1], publicKey, function (e, data) { - if (e) { - WARN(e, data); - return void Respond(e); - } - Respond(void 0, data); - }); - default: - return void Respond('UNSUPPORTED_RPC_CALL', msg); - } - }; - - handleMessage(true); - }; - - var rpc = function (ctx, data, respond) { - try { - return rpc0(ctx, data, respond); - } catch (e) { - console.log("Error from RPC with data " + JSON.stringify(data)); - console.log(e.stack); - } - }; - var updateLimitDaily = function () { - Quota.updateLimits(Env, config, undefined, function (e) { + Quota.updateLimits(Env, undefined, function (e) { if (e) { WARN('limitUpdate', e); } }); }; - Quota.applyCustomLimits(Env, config); + Quota.applyCustomLimits(Env); updateLimitDaily(); setInterval(updateLimitDaily, 24*3600*1000); @@ -451,7 +465,16 @@ RPC.create = function (config, cb) { Env.blobStore = blob; })); }).nThen(function () { - cb(void 0, rpc); + // XXX it's ugly that we pass ctx and Env separately + // when they're effectively the same thing... + cb(void 0, function (ctx, data, respond) { + try { + return rpc(Env, ctx, data, respond); + } catch (e) { + console.log("Error from RPC with data " + JSON.stringify(data)); + console.log(e.stack); + } + }); // expire old sessions once per minute // XXX allow for graceful shutdown Env.sessionExpirationInterval = setInterval(function () { From 0158ce6804c6c9fdcedec18a4214eaeee15cbc15 Mon Sep 17 00:00:00 2001 From: Weblate Date: Mon, 27 Jan 2020 10:12:28 +0000 Subject: [PATCH 034/172] Translated using Weblate (Catalan) Currently translated at 48.8% (585 of 1198 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/ca/ --- www/common/translations/messages.ca.json | 60 ++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/www/common/translations/messages.ca.json b/www/common/translations/messages.ca.json index 53e12630f..536f4b1bc 100644 --- a/www/common/translations/messages.ca.json +++ b/www/common/translations/messages.ca.json @@ -12,7 +12,7 @@ "media": "Multimèdia", "todo": "Tasques", "contacts": "Contactes", - "sheet": "Full (Beta)", + "sheet": "Full de càlcul", "teams": "Equips" }, "button_newpad": "Nou document", @@ -34,7 +34,7 @@ "inactiveError": "Donada la seva inactivitat, aquest document s'ha esborrat. Premeu Esc per crear un nou document.", "chainpadError": "Hi ha hagut un error crític mentre s'actualitzava el vostre contingut. Aquesta pàgina es manté en mode només de lectura per assegurar que no perdreu el que ja heu fet.
Premeu Esc per continuar veient aquest document o torneu a carregar la pàgina per provar de continuar editant-lo.", "invalidHashError": "El document que heu demanat té una adreça URL no vàlida.", - "errorCopy": " Encara podeu copiar el contingut en una altra ubicació prement Esc.
Un cop deixeu aquesta pàgina, desapareixerà per sempre!", + "errorCopy": " Encara podeu accedir al contingut prement Esc.
Un cop tanqueu aquesta finestra no hi podreu tornar a accedir.", "errorRedirectToHome": "Premeu Esc per tornar al vostre CryptDrive.", "newVersionError": "Hi ha una nova versió disponible de CryptPad.
Torneu a carregar la pàgina per utilitzar la versió nova o premeu Esc per accedir al vostre contingut en mode fora de línia.", "loading": "Carregant...", @@ -531,5 +531,59 @@ "settings_padSpellcheckTitle": "Correcció ortogràfica", "settings_padSpellcheckHint": "Aquesta opció us permet habilitar la correcció ortogràfica als documents de text. Les errades es subratllaran en vermell i haureu de mantenir apretada la tecla Ctrl o Meta mentre cliqueu el botó dret per veure les opcions correctes.", "settings_padSpellcheckLabel": "Activa la correcció ortogràfica", - "settings_creationSkip": "Salta la pantalla de creació de document" + "settings_creationSkip": "Salta la pantalla de creació de document", + "settings_creationSkipHint": "La pantalla de creació de documents ofereix noves opcions, donant-vos més control sobre les vostres dades. Tot i això, pot alentir una mica la feina afegint un pas addicional i, per això, teniu l'opció de saltar aquesta pantalla i utilitzar les opcions per defecte que hi ha seleccionades.", + "settings_creationSkipTrue": "Salta", + "settings_creationSkipFalse": "Mostra", + "settings_templateSkip": "Salta la finestra de selecció de plantilla", + "settings_templateSkipHint": "Quan genereu un document nou buit, si teniu desades plantilles per aquest tipus de document, apareix una finestra preguntant-vos si voleu utilitzar una plantilla. Aquí podeu triar si no voleu veure mai més la finestra i no utilitzar una plantilla.", + "settings_ownDriveTitle": "Habilita les darreres funcionalitats del compte", + "settings_ownDriveHint": "Per raons tècniques, els comptes antics no tenen accés a totes les funcionalitats noves. Si feu una actualització a un compte nou, preparareu el vostre CryptDrive per les properes funcionalitats sense interrompre la vostra activitat habitual.", + "settings_ownDriveButton": "Milloreu el vostre compte", + "settings_ownDriveConfirm": "Millorar el vostre compte porta una estona. Necessitareu tornar-vos a connectar en tots els vostres dispositius. Segur que ho voleu fer?", + "settings_ownDrivePending": "El vostre compte s'està posant al dia. No tanqueu ni torneu a carregar aquesta pàgina fins que el procés hagi acabat.", + "settings_changePasswordTitle": "Canvieu la contrasenya", + "settings_changePasswordHint": "Canvieu la contrasenya del vostre compte. Introduïu la contrasenya actual i confirmeu la nova escrivint-la dos cops.
Si l'oblideu, no podem recuperar la vostra contrasenya, aneu amb molt de compte!", + "settings_changePasswordButton": "Canvia la contrasenya", + "settings_changePasswordCurrent": "Contrasenya actual", + "settings_changePasswordNew": "Nova contrasenya", + "settings_changePasswordNewConfirm": "Confirma la nova contrasenya", + "settings_changePasswordConfirm": "Segur que voleu canviar la contrasenya? Necessitareu tornar-vos a connectar en tots els dispositius.", + "settings_changePasswordError": "Hi ha hagut una errada inesperada. Si no podeu iniciar la sessió o canviar la contrasenya, contacteu l'administració de CryptPad.", + "settings_changePasswordPending": "S'està actualitzant la contrasenya. Si us plau, no tanqueu ni carregueu de nou la pàgina fins que el procés s'hagi acabat.", + "settings_changePasswordNewPasswordSameAsOld": "La contrasenya nova cal que sigui diferent de l'actual.", + "settings_cursorColorTitle": "Color del cursor", + "settings_cursorColorHint": "Canvieu el color associat al vostre compte en els documents col·laboratius.", + "settings_cursorShareTitle": "Comparteix la posició del meu cursor", + "settings_cursorShareHint": "Podeu decidir si, als documents col·laboratius, voleu que la resta de persones vegin el vostre cursor.", + "settings_cursorShareLabel": "Comparteix la posició", + "settings_cursorShowTitle": "Mostra la posició del cursor de la resta", + "settings_cursorShowHint": "Podeu triar si, als documents col·laboratius, voleu veure el cursor de les altres persones.", + "settings_cursorShowLabel": "Mostra els cursors", + "upload_title": "Carrega fitxer", + "upload_type": "Tipus", + "upload_modal_title": "Opcions per carregar fitxers", + "upload_modal_filename": "Nom del fitxer (extensió {0} afegit automàticament)", + "upload_modal_owner": "Fitxer propi", + "uploadFolder_modal_title": "Opcions per carregar carpetes", + "uploadFolder_modal_filesPassword": "Fitxers de contrasenya", + "uploadFolder_modal_owner": "Fitxers propis", + "uploadFolder_modal_forceSave": "Deseu fitxers al vostre CryptDrive", + "upload_serverError": "Errada interna: ara mateix és impossible carregar el fitxer.", + "upload_uploadPending": "Ja teniu una càrrega en marxa. Voleu cancel·lar-la i carregar aquest altre fitxer?", + "upload_success": "El fitxer ({0}) ha estat carregat correctament i afegit al vostre CryptDrive.", + "upload_notEnoughSpace": "No hi ha prou espai al CryptDrive per aquest fitxer.", + "upload_notEnoughSpaceBrief": "No hi ha prou espai", + "upload_tooLarge": "Aquest fitxer supera la mida màxima permesa.", + "upload_tooLargeBrief": "El fitxer és massa gran", + "upload_choose": "Trieu un fitxer", + "upload_pending": "Pendent", + "upload_cancelled": "Cancel·lat", + "upload_name": "Nom del fitxer", + "upload_size": "Mida", + "upload_progress": "Procés", + "upload_mustLogin": "Cal que inicieu la sessió per carregar un fitxer", + "upload_up": "Carrega", + "download_button": "Desxifra i descarrega", + "download_mt_button": "Descarrega" } From 66ef508e0e7f784bfc8e70b3bb5ec4a098b71c9a Mon Sep 17 00:00:00 2001 From: Weblate Date: Mon, 27 Jan 2020 10:12:29 +0000 Subject: [PATCH 035/172] Translated using Weblate (English) Currently translated at 100.0% (1199 of 1199 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/en/ --- www/common/translations/messages.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/www/common/translations/messages.json b/www/common/translations/messages.json index d728f8742..62cb7a696 100644 --- a/www/common/translations/messages.json +++ b/www/common/translations/messages.json @@ -1294,5 +1294,6 @@ "oo_exportInProgress": "Export in progress", "oo_sheetMigration_loading": "Upgrading your spreadsheet to the latest version", "oo_sheetMigration_complete": "Updated version available, press OK to reload.", - "oo_sheetMigration_anonymousEditor": "Editing this spreadsheet is disabled for anonymous users until it is upgraded to the latest version by a registered user." + "oo_sheetMigration_anonymousEditor": "Editing this spreadsheet is disabled for anonymous users until it is upgraded to the latest version by a registered user.", + "imprint": "Legal notice" } From b6a6249eb44056c558ad6c5b43d440a5aac86c43 Mon Sep 17 00:00:00 2001 From: Weblate Date: Mon, 27 Jan 2020 10:12:29 +0000 Subject: [PATCH 036/172] Translated using Weblate (Finnish) Currently translated at 71.3% (855 of 1199 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/fi/ Translated using Weblate (Finnish) Currently translated at 71.4% (855 of 1198 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/fi/ --- www/common/translations/messages.fi.json | 147 ++++++++++++++++++++++- 1 file changed, 145 insertions(+), 2 deletions(-) diff --git a/www/common/translations/messages.fi.json b/www/common/translations/messages.fi.json index 365ea28b8..1df2e4d81 100644 --- a/www/common/translations/messages.fi.json +++ b/www/common/translations/messages.fi.json @@ -11,7 +11,7 @@ "media": "Media", "todo": "Tehtävälista", "contacts": "Yhteystiedot", - "sheet": "Taulukko (Beta)", + "sheet": "Taulukko", "teams": "Teams" }, "button_newpad": "Uusi Teksti-padi", @@ -876,8 +876,151 @@ "keywords": { "title": "Avainsanat", "pad": { - "q": "Mikä on padi?" + "q": "Mikä on padi?", + "a": "Padi on Etherpad-projektin popularisoima termi reaaliaikaiselle kollaboratiiviselle editorille.\nSe tarkoittaa selaimessa muokattavaa dokumenttia, jossa muiden käyttäjien tekemät muutokset näkyvät lähes välittömästi." + }, + "owned": { + "q": "Mikä on omistettu padi?", + "a": "Omistettu padi on padi, jolla on erityisesti määritelty omistaja, jonka palvelin tunnistaa julkisen salausavaimen perusteella. Padin omistaja voi poistaa omistamansa padit palvelimelta, jolloin muut yhteiskäyttäjät eivät voi enää käyttää niitä riippumatta siitä, olivatko ne tallennettuna heidän henkilökohtaisiin CryptDriveihinsa." + }, + "expiring": { + "q": "Mikä on vanheneva padi?", + "a": "Vanheneva padi on padi, jolle on määritelty vanhenemisajankohta, jolloin padi poistetaan automaattisesti palvelimelta. Vanhenevat padit voidaan määritellä säilymään minkä tahansa ajan yhdestä tunnista 100 kuukauteen. Vanheneva padi ja sen historia muuttuvat vanhenemishetkellä pysyvästi käyttökelvottomiksi, vaikka padia muokattaisiinkin silloin.

Jos padi on määritelty vanhenevaksi, voit tarkastaa sen vanhenemisajan padin ominaisuuksista joko CryptDrivessa padin kohdalla hiiren oikealla painikkeella aukeavasta valikosta tai käyttämällä Ominaisuudet-valikkoa sovelluksen työkalupalkista." + }, + "tag": { + "q": "Miten voin käyttää tunnisteita?", + "a": "Voit lisätä padeihin ja ladattuihin tiedostoihin tunnisteita CryptDrivessa tai käyttää Tunniste-painiketta () minkä tahansa editorin työkalupalkista. Hae padeja ja tiedostoja CryptDriven hakupalkista käyttämällä ristikkomerkillä alkavaa hakusanaa (esimerkiksi #crypto)." + }, + "template": { + "q": "Mikä on mallipohja?", + "a": "Mallipohja on padi, jolla voit määritellä luotavan padin oletussisällön luodessasi toista samantyyppistä padia. Voit muuttaa minkä tahansa olemassaolevan padin mallipohjaksi siirtämällä sen Mallipohjat-osastoon CryptDrivessasi. Voit myös tehdä padista mallipohjana käytettävän kopion klikkaamalla Mallipohja-painiketta () editorin työkalupalkista." + }, + "abandoned": { + "q": "Mikä on hylätty padi?", + "a": "Hylätty padi on padi, jota ei ole kiinnitetty yhdenkään rekisteröityneen käyttäjän CryptDriveen ja jota ei ole muokattu kuuteen kuukauteen. Hylätyt dokumentit poistetaan palvelimelta automaattisesti." + } + }, + "privacy": { + "title": "Yksityisyys", + "different": { + "q": "Miten CryptPad eroaa muista padeja tarjoavista palveluista?", + "a": "CryptPad salaa padeihin tekemäsi muutokset ennen niiden lähettämistä palvelimelle, joten emme voi lukea, mitä kirjoitat." + }, + "me": { + "q": "Mitä palvelin tietää minusta?", + "a": "Palvelimen ylläpitäjät näkevät CryptPadia käyttävien ihmisten IP-osoitteet. Emme pidä kirjaa siitä, mitkä osoitteet vierailevat missäkin padeissa. Tämä olisi kuitenkin teknisesti mahdollista, vaikka emme pääsekään tarkastelemaan padien salaamatonta sisältöä. Jos pelkäät meidän analysoivan näitä tietoja, on parasta olettaa meidän keräävän niitä, sillä emme voi todistaa, ettemme tee niin.

Keräämme käyttäjiltämme joitakin perustason telemetriatietoja, kuten käytetyn laitteen näytön koon ja tietoja useimmin käytetyistä painikkeista. Nämä auttavat meitä parantamaan CryptPadia, mutta jos et halua lähettää telemetriatietoja CryptPadille, voit jättäytyä pois tietojen keräämisestä ottamalla rastin pois Salli käyttäjäpalaute-ruudusta.

Pidämme kirjaa siitä, mitä padeja käyttäjät säilyttävät CryptDriveissaan pystyäksemme asettamaan tallennustilarajoituksia. Emme kuitenkaan tiedä näiden padien tyyppiä tai sisältöä. Tallennustilakiintiöt määritellään käyttäjien julkisten salausavainten perusteella, mutta emme yhdistä käyttäjien nimiä tai sähköpostiosoitteita näihin avaimiin.

Saadaksesi lisätietoja aiheesta voit tutustua kirjoittamaamme blogikirjoitukseen." + }, + "register": { + "q": "", + "a": "" + }, + "other": { + "q": "", + "a": "" + }, + "anonymous": { + "q": "", + "a": "" + }, + "policy": { + "q": "", + "a": "" + } + }, + "security": { + "pad_password": { + "q": "Mitä tapahtuu, kun suojaan padin tai kansion salasanalla?", + "a": "Voit suojata minkä tahansa padin tai jaetun kansion salasanalla luodessasi sen. Voit myös käyttää Ominaisuudet-valikkoa asettaaksesi, vaihtaaksesi tai poistaaksesi salasanan milloin tahansa.

Padien ja jaettujen kansioiden salasanat on tarkoitettu suojaamaan linkkiä jakaessasi sitä mahdollisesti turvattomien kanavien, kuten sähköpostin tai tekstiviestin kautta. Jos joku onnistuu kaappaamaan linkkisi, mutta ei tiedä sen salasanaa, ei hän pääse lukemaan dokumenttiasi.

Kun jaat sisältöä CryptPadin sisällä yhteystietojesi tai tiimiesi kanssa, tiedonsiirto on salattua ja oletamme, että haluat heidän pääsevän käyttämään dokumenttiasi. Siksi salasana tallennetaan ja lähetetään padin mukana jakaessasi sitä CryptPadin sisällä. Vastaanottajalta tai sinulta itseltäsi ei pyydetä salasanaa dokumenttia avatessa." + }, + "title": "", + "proof": { + "q": "", + "a": "" + }, + "why": { + "q": "", + "a": "" + }, + "compromised": { + "q": "", + "a": "" + }, + "crypto": { + "q": "", + "a": "" + } + }, + "usability": { + "title": "Käytettävyys", + "register": { + "q": "Mitä hyötyä rekisteröitymisestä on minulle?", + "a": "Rekisteröityneille käyttäjille on tarjolla joitakin toimintoja, jotka eivät ole saatavilla rekisteröitymättömille käyttäjille. Löydät nämä toiminnot luomastamme kaaviosta." + }, + "share": { + "q": "Miten jaan salattuja padeja kavereideni kanssa?", + "a": "CryptPad laittaa URL-osoitteessa padisi salaisen salausavaimen #-merkin jälkeen. Tämän merkin jälkeen laitettuja tietoja ei lähetetä palvelimelle, joten emme pääse koskaan käyttämään salausavaimiasi. Jakaessasi linkin padiin jaat oikeuden lukea ja käyttää sitä." + }, + "remove": { + "q": "Poistin padin tai tiedoston CryptDrivestani, mutta sen sisältö on yhä käytettävissä. Miten voin poistaa sen?", + "a": "Ainoastaan omistettuja padeja (otettu käyttöön helmikuussa 2018) voi poistaa. Lisäksi nämä padit voi poistaa ainoastaan niiden omistaja eli henkilö, joka alun perin loi kyseisen padin. Jos et ole luonut kyseistä padia, joudut pyytämään sen omistajaa poistamaan sen puolestasi. Omistamiesi padien poistaminen onnistuu CryptDrivessa klikkaamalla padia hiiren oikealla painikkeella ja valitsemalla Poista palvelimelta." + }, + "forget": { + "q": "Mitä tapahtuu, jos unohdan salasanani?", + "a": "Valitettavasti se, että pystyisimme palauttamaan käyttöoikeuden salattuihin padeihisi tarkoittaisi myös sitä, että pääsisimme itse käsiksi niiden sisältöön. Jos et kirjoittanut käyttäjätunnustasi ja salasanaasi ylös etkä muista kumpaakaan, voit mahdollisesti palauttaa padisi selaimesi historiaa suodattamalla." + }, + "change": { + "q": "Entä jos haluan vaihtaa salasanani?", + "a": "Voit vaihtaa CryptPad-salasanasi Tilin asetukset-sivulta." + }, + "devices": { + "q": "Olen kirjautunut sisään kahdella laitteella, ja näen kaksi eri CryptDrivea. Miten tämä on mahdollista?", + "a": "On todennäköistä, että olet rekisteröitynyt samalla käyttäjänimellä kahdesti eri salasanoja käyttäen. CryptPad-palvelin tunnistaa sinut kryptografisen allekirjoituksesi perusteella käyttäjänimen sijaan, joten se ei voi estää muita rekisteröitymästä samalla käyttäjänimellä. Tästä johtuen jokaisella käyttäjätilillä on ainutlaatuinen käyttäjänimen ja salasanan yhdistelmä. Sisäänkirjautuneet käyttäjät voivat nähdä käyttäjänimensä Asetukset-sivun ylälaidassa." + }, + "folder": { + "q": "Voinko jakaa kokonaisia kansioita CryptDrivestani?", + "a": "Kyllä, voit luoda jaetun kansion CryptDrivestasi ja jakaa kerralla kaikki sen sisältämät padit." + }, + "feature": { + "q": "Voitteko lisätä CryptPadiin tarvitsemani ominaisuuden?", + "a": "Monet CryptPadin ominaisuuksista ovat olemassa, koska käyttäjämme ovat toivoneet niitä. Yhteystiedot-sivumme kertoo, millä tavoin meihin saa yhteyden.

Valitettavasti emme voi taata, että pystymme toteuttamaan kaikki käyttäjiemme ehdotukset. Jos jokin tietty ominaisuus on kriittinen organisaatiosi kannalta, voit sponsoroida kehitystä varmistaaksesi sen toteutumisen. Ota yhteyttä osoitteeseen sales@cryptpad.fr saadaksesi lisätietoja.

Vaikka kehitystyön sponsorointi ei olisikaan mahdollista, olemme silti kiinnostuneita palautteesta, joka auttaa meitä parantamaan CryptPadia. Ota meihin milloin tahansa yhteyttä yllä luetelluilla tavoilla." + } + }, + "other": { + "title": "Muita kysymyksiä", + "pay": { + "q": "Miksi minun täytyisi maksaa, kun niin monet toiminnot ovat ilmaisia?", + "a": "Annamme tukijoillemme lisätallennustilaa ja mahdollisuuden kasvattaa kavereiden tallennustilakiintiöitä (lue lisää).

Näiden lyhytaikaisten etujen lisäksi premium-tilaus auttaa rahoittamaan CryptPadin jatkuvaa, aktiivista kehitystyötä. Tähän kuuluu bugien korjaamista, uusien ominaisuuksien lisäämistä ja CryptPad-instanssien pystyttämisen ja ylläpidon helpottamista. Lisäksi autat näyttämään muille palveluntarjoajille, että ihmiset ovat valmiita tukemaan yksityisyyttä parantavia teknologioita. Toivomme, että käyttäjätietojen myymiseen perustuvat liiketoimintamallit jäävät lopulta menneeseen.

Lopuksi, tarjoamme suurimman osan CryptPadin toiminnallisuudesta ilmaiseksi, koska uskomme yksityisyyden kuuluvan kaikille - ei vain niille, joilla on varaa maksaa siitä. Tukemalla meitä autat tarjoamaan heikommassa asemassa oleville väestöille pääsyn näihin peruspalveluihin." + }, + "goal": { + "q": "Mitkä ovat tavoitteenne?", + "a": "Kehittämällä yksityisyyttä kunnioittavaa kollaboraatioteknologiaa toivomme nostavamme käyttäjien odotuksia pilvipalveluiden yksityisyyden suhteen. Toivomme, että työmme rohkaisee muita palveluntarjoajia pyrkimään samaan tai parempaan lopputulokseen. Optimismistamme huolimatta tiedämme, että suuri osa webistä rahoitetaan kohdistetulla mainonnalla. Tehtävää on paljon enemmän, kuin mihin pystymme yksin - arvostamme yhteisömme tarjoamaa mainostusta, tukea ja panosta tavoitteidemme saavuttamisessa." + }, + "jobs": { + "q": "Etsittekö työntekijöitä?", + "a": "Kyllä! Esittäydy meille sähköpostilla osoitteeseen jobs@xwiki.com." + }, + "host": { + "q": "Voitteko auttaa minua perustamaan oman CryptPad-instanssini?", + "a": "Tarjoamme mielellämme tukea organisaatiosi sisäiselle CryptPad-instanssille. Ota yhteyttä osoitteeseen sales@cryptpad.fr saadaksesi lisätietoja." + }, + "revenue": { + "q": "Kuinka voin osallistua tulojen jakamiseen?", + "a": "Jos ylläpidät omaa CryptPad-instanssiasi, haluaisit ottaa käyttöön maksulliset käyttäjätilit ja jakaa tulot CryptPadin kehittäjien kanssa, palvelimesi täytyy määritellä kumppanipalveluksi.

CryptPad-asennushakemistosi config.example.js-tiedostosta pitäisi löytyä ohjeet tämän palvelun käyttöönottoon. Sinun tulee myös ottaa yhteyttä osoitteeseen sales@cryptpad.fr varmistaaksesi, että palvelimesi HTTPS-määritykset ovat kunnossa ja sopiaksesi käytettävistä maksutavoista." } } + }, + "policy_howweuse_p1": "Käytämme näitä tietoja suunnitellaksemme CryptPadin mainostusta ja arvioidaksemme aiempien kampanjoiden onnistumista. Sijaintitietosi puolestaan kertovat meille, mitä kieliä CryptPadin tulisi mahdollisesti tukea englannin lisäksi.", + "tos_title": "", + "tos_legal": "", + "tos_availability": "", + "tos_e2ee": "", + "tos_logs": "", + "tos_3rdparties": "", + "four04_pageNotFound": "", + "updated_0_header_logoTitle": "", + "header_logoTitle": "", + "header_homeTitle": "", + "help": { + "title": "" } } From 6f6bbaf75ecc5c9387001f47d1b672cdfcba364e Mon Sep 17 00:00:00 2001 From: Weblate Date: Mon, 27 Jan 2020 10:12:29 +0000 Subject: [PATCH 037/172] Translated using Weblate (French) Currently translated at 100.0% (1199 of 1199 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/fr/ --- www/common/translations/messages.fr.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/www/common/translations/messages.fr.json b/www/common/translations/messages.fr.json index 014837192..9abea1c41 100644 --- a/www/common/translations/messages.fr.json +++ b/www/common/translations/messages.fr.json @@ -1294,5 +1294,6 @@ "oo_exportInProgress": "Exportation en cours", "oo_sheetMigration_loading": "Mise à jour de la feuille de calcul", "oo_sheetMigration_complete": "Version mise à jour disponible, appuyez sur OK pour recharger.", - "oo_sheetMigration_anonymousEditor": "L'édition de cette feuille de calcul est désactivée pour les utilisateurs anonymes jusqu'à ce qu'elle soit mise à jour par un utilisateur enregistré." + "oo_sheetMigration_anonymousEditor": "L'édition de cette feuille de calcul est désactivée pour les utilisateurs anonymes jusqu'à ce qu'elle soit mise à jour par un utilisateur enregistré.", + "imprint": "Mentions légales" } From 27f864128d745e88b1e4af023af28974a294db68 Mon Sep 17 00:00:00 2001 From: Weblate Date: Mon, 27 Jan 2020 10:12:30 +0000 Subject: [PATCH 038/172] Translated using Weblate (German) Currently translated at 100.0% (1199 of 1199 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/de/ Translated using Weblate (German) Currently translated at 100.0% (1199 of 1199 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/de/ Translated using Weblate (German) Currently translated at 100.0% (1199 of 1199 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/de/ --- www/common/translations/messages.de.json | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/www/common/translations/messages.de.json b/www/common/translations/messages.de.json index 679252b23..6837b3d8e 100644 --- a/www/common/translations/messages.de.json +++ b/www/common/translations/messages.de.json @@ -12,7 +12,7 @@ "media": "Medien", "todo": "Aufgaben", "contacts": "Kontakte", - "sheet": "Tabelle (Beta)", + "sheet": "Tabelle", "teams": "Teams" }, "button_newpad": "Neues Rich-Text-Pad", @@ -92,7 +92,7 @@ "exportButtonTitle": "Exportiere dieses Pad in eine lokale Datei", "exportPrompt": "Wie möchtest du die Datei nennen?", "changeNamePrompt": "Ändere deinen Namen (oder lasse dieses Feld leer, um anonym zu bleiben): ", - "user_rename": "Bearbeite deinen Anzeigename", + "user_rename": "Anzeigename ändern", "user_displayName": "Anzeigename", "user_accountName": "Kontoname", "clickToEdit": "Zum Bearbeiten klicken", @@ -278,8 +278,8 @@ "userlist_addAsFriendTitle": "Benutzer \"{0}\" eine Freundschaftsanfrage senden", "contacts_title": "Kontakte", "contacts_addError": "Fehler bei dem Hinzufügen des Kontakts zur Liste", - "contacts_added": "Verbindungseinladung angenommen.", - "contacts_rejected": "Verbindungseinladung abgelehnt", + "contacts_added": "Kontaktanfrage akzeptiert.", + "contacts_rejected": "Kontaktanfrage abgelehnt", "contacts_request": "Benutzer {0} möchte dich als Kontakt hinzufügen. Annehmen?", "contacts_send": "Senden", "contacts_remove": "Diesen Kontakt entfernen", @@ -1100,7 +1100,7 @@ "support_formMessage": "Gib deine Nachricht ein...", "support_cat_tickets": "Vorhandene Tickets", "support_listTitle": "Support-Tickets", - "support_listHint": "Hier ist die Liste der an die Administratoren gesendeten Tickets und der dazugehörigen Antworten. Ein geschlossenes Ticket kann nicht wieder geöffnet werden, du musst ein Ticket eröffnen. Du kannst geschlossene Tickets ausblenden, aber sie werden weiterhin für die Administratoren sichtbar sein.", + "support_listHint": "Hier ist die Liste der an die Administratoren gesendeten Tickets und der dazugehörigen Antworten. Ein geschlossenes Ticket kann nicht wieder geöffnet werden, du kannst jedoch ein neues Ticket eröffnen. Du kannst geschlossene Tickets ausblenden, aber sie werden weiterhin für die Administratoren sichtbar sein.", "support_answer": "Antworten", "support_close": "Ticket schließen", "support_remove": "Ticket entfernen", @@ -1294,5 +1294,6 @@ "oo_exportInProgress": "Export wird durchgeführt", "oo_sheetMigration_loading": "Deine Tabelle wird auf die neueste Version aktualisiert", "oo_sheetMigration_complete": "Eine aktualisierte Version ist verfügbar. Klicke auf OK, um neu zu laden.", - "oo_sheetMigration_anonymousEditor": "Die Bearbeitung dieser Tabelle ist für anonyme Benutzer deaktiviert, bis sie von einem registrierten Benutzer auf die neueste Version aktualisiert wird." + "oo_sheetMigration_anonymousEditor": "Die Bearbeitung dieser Tabelle ist für anonyme Benutzer deaktiviert, bis sie von einem registrierten Benutzer auf die neueste Version aktualisiert wird.", + "imprint": "Impressum" } From b8ec7178da162c5dcc8f0942ab2eee9fd7c4b779 Mon Sep 17 00:00:00 2001 From: Weblate Date: Mon, 27 Jan 2020 10:12:30 +0000 Subject: [PATCH 039/172] Translated using Weblate (Italian) Currently translated at 41.7% (499 of 1198 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/it/ --- www/common/translations/messages.it.json | 42 +++++++++++++----------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/www/common/translations/messages.it.json b/www/common/translations/messages.it.json index 4d8b28b5d..2a8211f03 100644 --- a/www/common/translations/messages.it.json +++ b/www/common/translations/messages.it.json @@ -1,8 +1,8 @@ { - "main_title": "CryptPad: Editor collaborativo in tempo reale, zero knowledge", + "main_title": "CryptPad: Editor zero knowledge collaborativo in tempo reale", "type": { "pad": "Testo", - "code": "Code", + "code": "Codice", "poll": "Sondaggio", "kanban": "Kanban", "slide": "Presentazione", @@ -10,13 +10,13 @@ "whiteboard": "Lavagna", "file": "File", "media": "Media", - "todo": "Todo", + "todo": "Promemoria", "contacts": "Contatti", - "sheet": "Fogli (Beta)", + "sheet": "Fogli", "teams": "Team" }, "button_newpad": "Nuovo pad di Testo", - "button_newcode": "Nuovo pad di Code", + "button_newcode": "Nuovo pad di Codice", "button_newpoll": "Nuovo Sondaggio", "button_newslide": "Nuova Presentazione", "button_newwhiteboard": "Nuova Lavagna", @@ -34,7 +34,7 @@ "inactiveError": "Questo pad è stato cancellato per inattività. Premi Esc per creare un nuovo pad.", "chainpadError": "Si è verificato un errore critico nell'aggiornamento del tuo contenuto. Questa pagina è in modalità solo lettura per assicurarci che non perderai il tuo lavoro..
Premi Esc per continuare a visualizzare questo pad, o ricarica la pagina per provare a modificarlo di nuovo.", "invalidHashError": "Il documento richiesto ha un URL non valido.", - "errorCopy": " Puoi ancora copiare il contenuto altrove premendo Esc.
Una volta abbandonata questa pagina, non sarà possibile recuperarlo!", + "errorCopy": " Puoi ancora accedere al contenuto premendo Esc.
Una volta chiusa questa finestra, non sarà possibile accedere di nuovo.", "errorRedirectToHome": "Premi Esc per essere reindirizzato al tuo CryptDrive.", "newVersionError": "Una nuova versione di CryptPad è disponibile.
Ricarica per usare la nuova versione, o premi Esc per accedere al contenuto in modalità offline.", "loading": "Caricamento...", @@ -447,19 +447,19 @@ "settings_exportTitle": "Esporta il tuo CryptDrive", "settings_exportDescription": "Per favore attendi mentre scarichiamo e decriptiamo i tuoi documenti. Potrebbe richiedere qualche minuto. Chiudere la finestra interromperà il processo.", "settings_exportFailed": "Se il pad richiede più di un minuto per essere scaricato, non sarà incluso nell'export. Un link a qualsiasi pad non esportato sarà mostrato.", - "settings_exportWarning": "", - "settings_exportCancel": "", - "settings_export_reading": "", - "settings_export_download": "", - "settings_export_compressing": "", - "settings_export_done": "", - "settings_exportError": "", - "settings_exportErrorDescription": "", - "settings_exportErrorEmpty": "", - "settings_exportErrorMissing": "", - "settings_exportErrorOther": "", + "settings_exportWarning": "Nota bene: questo strumento è ancora in versione beta e può presentare problemi di scalabilità. Per migliorare le prestazioni, è consigliabile lasciare attiva questa tab.", + "settings_exportCancel": "Sei sicuro di voler cancellare l'export? Dovrai iniziare da capo la prossima volta.", + "settings_export_reading": "Lettura del tuo CryptDrive in corso...", + "settings_export_download": "Scaricamento e decriptazione dei tuoi documenti in corso...", + "settings_export_compressing": "Compressione dei dati in corso...", + "settings_export_done": "Il tuo download è pronto!", + "settings_exportError": "Visualizza errori", + "settings_exportErrorDescription": "Non siamo riusciti ad aggiungere i seguenti documenti all'export:", + "settings_exportErrorEmpty": "Questo documento non può essere esportato (contenuto vuoto o invalido).", + "settings_exportErrorMissing": "Questo documento non è stato trovato nei nostri server (scaduto o rimosso dal suo proprietario)", + "settings_exportErrorOther": "È accaduto un errore durante l'esportazione di questo documento: {0}", "settings_resetNewTitle": "Pulisci CryptDrive", - "settings_resetButton": "", + "settings_resetButton": "Rimuovi", "settings_reset": "Rimuovi tutti i file e le cartelle dal tuo CryptDrive", "settings_resetPrompt": "", "settings_resetDone": "", @@ -513,5 +513,9 @@ }, "readme_cat3_l1": "Con l'editor di codice di CryptPad, puoi collaborare su linguaggi di programmazione come Javascript e linguaggi di markup come HTML o Markdown", "settings_codeSpellcheckLabel": "Abilita la revisione ortografica nell'editor di codice", - "team_inviteLinkError": "Si è verificato un errore durante la creazione del link." + "team_inviteLinkError": "Si è verificato un errore durante la creazione del link.", + "register_emailWarning1": "Puoi farlo se vuoi, ma non verrà inviato ai nostri server.", + "register_emailWarning2": "Non sarai in grado di resettare la tua password usando la tua email, a differenza di come puoi fare con molti altri servizi.", + "register_emailWarning3": "Se hai capito, ma intendi comunque usare la tua email come nome utente, clicca OK.", + "oo_sheetMigration_anonymousEditor": "Le modifiche da parte di utenti anonimi a questo foglio di calcolo sono disabilitate finchè un utente registrato non lo aggiorna all'ultima versione." } From 0ad96e0966faf660cc3ba8a62736b3e370e2f44a Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 27 Jan 2020 12:18:25 +0100 Subject: [PATCH 040/172] Hide the crypto keys from the hash --- www/common/common-hash.js | 42 ++++++++- www/common/cryptpad-common.js | 40 ++++++--- www/common/onlyoffice/main.js | 10 +++ www/common/outer/async-store.js | 39 +++++++- www/common/outer/store-rpc.js | 1 + www/common/sframe-app-outer.js | 11 +++ www/common/sframe-common-outer.js | 143 ++++++++++++++++++++++++------ www/poll/main.js | 11 +++ 8 files changed, 251 insertions(+), 46 deletions(-) diff --git a/www/common/common-hash.js b/www/common/common-hash.js index 90ccf805c..b92aea475 100644 --- a/www/common/common-hash.js +++ b/www/common/common-hash.js @@ -60,6 +60,18 @@ var factory = function (Util, Crypto, Nacl) { return '/2/' + secret.type + '/view/' + Crypto.b64RemoveSlashes(data.viewKeyStr) + '/' + pass; } }; + Hash.getHiddenHashFromKeys = function (type, secret, opts) { + var mode = (secret.keys && secret.keys.editKeyStr) ? 'edit' : 'view'; + var pass = secret.password ? 'p/' : ''; + var hash = '/2/' + secret.type + '/' + mode + '/' + secret.channel + '/' + pass; + var href = '/' + type + '/#' + hash; + var parsed = Hash.parsePadUrl(href); + if (parsed.hashData && parsed.hashData.getHash) { + return parsed.hashData.getHash(opts || {}); + } + return hash; + }; + var getFileHashFromKeys = Hash.getFileHashFromKeys = function (secret) { var version = secret.version; var data = secret.keys; @@ -192,6 +204,13 @@ Version 1 if (opts.present) { hash += 'present/'; } return hash; }; + parsed.getOptions = function () { + return { + embed: parsed.embed, + present: parsed.present, + ownerKey: parsed.ownerKey + }; + }; return parsed; } if (hashArr[1] && hashArr[1] === '2') { // Version 2 @@ -221,6 +240,13 @@ Version 1 if (opts.present) { hash += 'present/'; } return hash; }; + parsed.getOptions = function () { + return { + embed: parsed.embed, + present: parsed.present, + ownerKey: parsed.ownerKey + }; + }; return parsed; } return parsed; @@ -256,6 +282,13 @@ Version 1 if (opts.present) { hash += 'present/'; } return hash; }; + parsed.getOptions = function () { + return { + embed: parsed.embed, + present: parsed.present, + ownerKey: parsed.ownerKey + }; + }; return parsed; } return parsed; @@ -309,6 +342,10 @@ Version 1 url += '#' + hash; return url; }; + ret.getOptions = function () { + if (!ret.hashData || !ret.hashData.getOptions) { return {}; } + return ret.hashData.getOptions(); + }; if (!/^https*:\/\//.test(href)) { idx = href.indexOf('/#'); @@ -497,8 +534,9 @@ Version 1 if (typeof(parsed.hashData.version) === "undefined") { return; } // pads and files should have a base64 (or hex) key if (parsed.hashData.type === 'pad' || parsed.hashData.type === 'file') { - if (!parsed.hashData.key) { return; } - if (!/^[a-zA-Z0-9+-/=]+$/.test(parsed.hashData.key)) { return; } + if (!parsed.hashData.key && !parsed.hashData.channel) { return; } + if (parsed.hashData.key && !/^[a-zA-Z0-9+-/=]+$/.test(parsed.hashData.key)) { return; } + if (parsed.hashData.channel && !/^[a-f0-9]{32,34}$/.test(parsed.hashData.channel)) { return; } } } return true; diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index a0057c59a..0be93e14c 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -49,6 +49,12 @@ define([ account: {}, }; + // Store the href in memory + // This is a placeholder value overriden in common.ready from sframe-common-outer + var currentPad = { + href: window.location.href + }; + // COMMON common.getLanguage = function () { return Messages._languageUsed; @@ -374,7 +380,7 @@ define([ common.getMetadata = function (cb) { - var parsed = Hash.parsePadUrl(window.location.href); + var parsed = Hash.parsePadUrl(currentPad.href); postMessage("GET_METADATA", parsed && parsed.type, function (obj) { if (obj && obj.error) { return void cb(obj.error); } cb(null, obj); @@ -394,7 +400,7 @@ define([ common.setPadAttribute = function (attr, value, cb, href) { cb = cb || function () {}; - href = Hash.getRelativeHref(href || window.location.href); + href = Hash.getRelativeHref(href || currentPad.href); postMessage("SET_PAD_ATTRIBUTE", { href: href, attr: attr, @@ -405,7 +411,7 @@ define([ }); }; common.getPadAttribute = function (attr, cb, href) { - href = Hash.getRelativeHref(href || window.location.href); + href = Hash.getRelativeHref(href || currentPad.href); if (!href) { return void cb('E404'); } @@ -505,7 +511,7 @@ define([ }; common.saveAsTemplate = function (Cryptput, data, cb) { - var p = Hash.parsePadUrl(window.location.href); + var p = Hash.parsePadUrl(currentPad.href); if (!p.type) { return; } // PPP: password for the new template? var hash = Hash.createRandomHash(p.type); @@ -543,7 +549,7 @@ define([ var href = data.href; var parsed = Hash.parsePadUrl(href); - var parsed2 = Hash.parsePadUrl(window.location.href); + var parsed2 = Hash.parsePadUrl(currentPad.href); if(!parsed) { throw new Error("Cannot get template hash"); } postMessage("INCREMENT_TEMPLATE_USE", href); @@ -601,7 +607,7 @@ define([ var fileHost = Config.fileHost || window.location.origin; var data = common.fromFileData; var parsed = Hash.parsePadUrl(data.href); - var parsed2 = Hash.parsePadUrl(window.location.href); + var parsed2 = Hash.parsePadUrl(currentPad.href); var hash = parsed.hash; var name = data.title; var secret = Hash.getSecrets('file', hash, data.password); @@ -660,7 +666,7 @@ define([ // Forget button common.moveToTrash = function (cb, href) { - href = href || window.location.href; + href = href || currentPad.href; postMessage("MOVE_TO_TRASH", { href: href }, cb); }; @@ -668,7 +674,7 @@ define([ common.setPadTitle = function (data, cb) { if (!data || typeof (data) !== "object") { return cb ('Data is not an object'); } - var href = data.href || window.location.href; + var href = data.href || currentPad.href; var parsed = Hash.parsePadUrl(href); if (!parsed.hash) { return cb ('Invalid hash'); } data.href = parsed.getUrl({present: parsed.present}); @@ -698,7 +704,7 @@ define([ if (obj.error !== "EAUTH") { console.log("unable to set pad title"); } return void cb(obj.error); } - cb(); + cb(null, obj); }); }; @@ -755,6 +761,13 @@ define([ cb(void 0, data); }); }; + // Get data about a given channel: use with hidden hashes + common.getPadDataFromChannel = function (obj, cb) { + if (!obj || !obj.channel || !obj.edit) { return void cb('EINVAL'); } + postMessage("GET_PAD_DATA_FROM_CHANNEL", obj, function (data) { + cb(void 0, data); + }); + }; // Admin @@ -1608,7 +1621,7 @@ define([ hashes = Hash.getHashes(secret); return void cb(null, hashes); } - var parsed = Hash.parsePadUrl(window.location.href); + var parsed = Hash.parsePadUrl(currentPad.href); if (!parsed.type || !parsed.hashData) { return void cb('E_INVALID_HREF'); } hashes = Hash.getHashes(secret); @@ -1679,7 +1692,7 @@ define([ LocalStore.logout(); // redirect them to log in, and come back when they're done. - sessionStorage.redirectTo = window.location.href; + sessionStorage.redirectTo = currentPad.href; window.location.href = '/login/'; }; @@ -1780,6 +1793,11 @@ define([ return function (f, rdyCfg) { rdyCfg = rdyCfg || {}; + + if (rdyCfg.currentPad) { + currentPad = rdyCfg.currentPad; + } + if (initialized) { return void setTimeout(function () { f(void 0, env); }); } diff --git a/www/common/onlyoffice/main.js b/www/common/onlyoffice/main.js index b3a896360..600901872 100644 --- a/www/common/onlyoffice/main.js +++ b/www/common/onlyoffice/main.js @@ -9,6 +9,7 @@ define([ var requireConfig = RequireConfig(); // Loaded in load #2 + var hash, href; nThen(function (waitFor) { DomReady.onReady(waitFor()); }).nThen(function (waitFor) { @@ -19,6 +20,13 @@ define([ }; window.rc = requireConfig; window.apiconf = ApiConfig; + + // Hidden hash + hash = window.location.hash; + href = window.location.href; + if (window.history && window.history.replaceState) { + window.history.replaceState({}, window.document.title, '#'); + } document.getElementById('sbox-iframe').setAttribute('src', ApiConfig.httpSafeOrigin + window.location.pathname + 'inner.html?' + requireConfig.urlArgs + '#' + encodeURIComponent(JSON.stringify(req))); @@ -144,6 +152,8 @@ define([ }); }; SFCommonO.start({ + hash: hash, + href: href, type: 'oo', useCreationScreen: true, addData: addData, diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index 681f1d575..4b2fda935 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -1016,8 +1016,12 @@ define([ if (title.trim() === "") { title = UserObject.getDefaultName(p); } - if (AppConfig.disableAnonymousStore && !store.loggedIn) { return void cb(); } - if (p.type === "debug") { return void cb(); } + if (AppConfig.disableAnonymousStore && !store.loggedIn) { + return void cb({ notStored: true }); + } + if (p.type === "debug") { + return void cb({ notStored: true }); + } var channelData = Store.channels && Store.channels[channel]; @@ -1108,7 +1112,7 @@ define([ postMessage(clientId, "AUTOSTORE_DISPLAY_POPUP", { autoStore: autoStore }); - return void cb(); + return void cb({ notStored: true }); } else { var roHref; if (h.mode === "view") { @@ -1187,7 +1191,9 @@ define([ }); cb(list); }; - // Get the first pad we can find in any of our managers and return its file data + + // Get the first pad we can find in any of our drives and return its file data + // NOTE: This is currently only used for template: this won't search inside shared folders Store.getPadData = function (clientId, id, cb) { var res = {}; getAllStores().some(function (s) { @@ -1199,6 +1205,31 @@ define([ cb(res); }; + Store.getPadDataFromChannel = function (clientId, obj, cb) { + var channel = obj.channel; + var edit = obj.edit; + var res; + var viewRes; + getAllStores().some(function (s) { + var chans = s.manager.findChannel(channel); + if (!Array.isArray(chans)) { return; } + return chans.some(function (pad) { + if (!pad || !pad.data) { return; } + var data = pad.data; + // We've found a match: return the value and stop the loops + if ((edit && data.href) || (!edit && data.roHref)) { + res = data; + return true; + } + // We've found a weaker match: store it for now + if (edit && !viewRes && data.roHref) { + viewRes = data; + } + }); + }); + // Call back with the best value we can get + cb(res || viewRes || {}); + }; // Messaging (manage friends from the userlist) Store.answerFriendRequest = function (clientId, obj, cb) { diff --git a/www/common/outer/store-rpc.js b/www/common/outer/store-rpc.js index 41963402b..41a3f7a0e 100644 --- a/www/common/outer/store-rpc.js +++ b/www/common/outer/store-rpc.js @@ -50,6 +50,7 @@ define([ GET_TEMPLATES: Store.getTemplates, GET_SECURE_FILES_LIST: Store.getSecureFilesList, GET_PAD_DATA: Store.getPadData, + GET_PAD_DATA_FROM_CHANNEL: Store.getPadDataFromChannel, GET_STRONGER_HASH: Store.getStrongerHash, INCREMENT_TEMPLATE_USE: Store.incrementTemplateUse, GET_SHARED_FOLDER: Store.getSharedFolder, diff --git a/www/common/sframe-app-outer.js b/www/common/sframe-app-outer.js index cc4d5fcb3..563430c42 100644 --- a/www/common/sframe-app-outer.js +++ b/www/common/sframe-app-outer.js @@ -8,6 +8,7 @@ define([ ], function (nThen, ApiConfig, DomReady, RequireConfig, SFCommonO) { var requireConfig = RequireConfig(); + var hash, href; nThen(function (waitFor) { DomReady.onReady(waitFor()); }).nThen(function (waitFor) { @@ -18,6 +19,14 @@ define([ }; window.rc = requireConfig; window.apiconf = ApiConfig; + + // Hidden hash + hash = window.location.hash; + href = window.location.href; + if (window.history && window.history.replaceState) { + window.history.replaceState({}, window.document.title, '#'); + } + document.getElementById('sbox-iframe').setAttribute('src', ApiConfig.httpSafeOrigin + window.location.pathname + 'inner.html?' + requireConfig.urlArgs + '#' + encodeURIComponent(JSON.stringify(req))); @@ -36,6 +45,8 @@ define([ window.addEventListener('message', onMsg); }).nThen(function (/*waitFor*/) { SFCommonO.start({ + hash: hash, + href: href, useCreationScreen: true, messaging: true }); diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index ca84bc637..538fb4019 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -30,6 +30,11 @@ define([ var password; var initialPathInDrive; + var currentPad = { + href: cfg.href || window.location.href, + hash: cfg.hash || window.location.hash + }; + nThen(function (waitFor) { // Load #2, the loading screen is up so grab whatever you need... require([ @@ -134,11 +139,12 @@ define([ }); } }), { - driveEvents: cfg.driveEvents + driveEvents: cfg.driveEvents, + currentPad: currentPad }); })); }).nThen(function (waitFor) { - if (!Utils.Hash.isValidHref(window.location.href)) { + if (!Utils.Hash.isValidHref(currentPad.href)) { waitFor.abort(); return void sframeChan.event('EV_LOADING_ERROR', 'INVALID_HASH'); } @@ -171,11 +177,12 @@ define([ }); })); } else { - var parsed = Utils.Hash.parsePadUrl(window.location.href); + var parsed = Utils.Hash.parsePadUrl(currentPad.href); var todo = function () { - secret = Utils.secret = Utils.Hash.getSecrets(parsed.type, void 0, password); + secret = Utils.secret = Utils.Hash.getSecrets(parsed.type, parsed.hash, password); Cryptpad.getShareHashes(secret, waitFor(function (err, h) { hashes = h; + /* XXX this won't happen again: we don't need to update the rendered hash if (password && !parsed.hashData.password) { var ohc = window.onhashchange; window.onhashchange = function () {}; @@ -183,6 +190,7 @@ define([ window.onhashchange = ohc; ohc({reset: true}); } + */ })); }; @@ -241,13 +249,13 @@ define([ if (parsed.type === "file") { // `isNewChannel` doesn't work for files (not a channel) // `getFileSize` is not adapted to channels because of metadata - Cryptpad.getFileSize(window.location.href, password, function (e, size) { + Cryptpad.getFileSize(currentPad.href, password, function (e, size) { next(e, size === 0); }); return; } // Not a file, so we can use `isNewChannel` - Cryptpad.isNewChannel(window.location.href, password, next); + Cryptpad.isNewChannel(currentPad.href, password, next); }); sframeChan.event("EV_PAD_PASSWORD", cfg); }; @@ -257,7 +265,60 @@ define([ var passwordCfg = { value: '' }; + + // Hidden hash: can't find the channel in our drives: abort + var noPadData = function (err) { + console.error(err); + // XXX Display error screen in inner + }; + // Hidden hash: can't find requestd edit URL in our drives: ask + var badPadData = function (cb) { + // If we requested edit but we only know view: ??? + setTimeout(function () { + cb(true); + }); // XXX ask in inner? + }; + + var newHref; nThen(function (w) { + if (!parsed.hashData.key && parsed.hashData.channel) { + Cryptpad.getPadDataFromChannel({ + channel: parsed.hashData.channel, + edit: parsed.hashData.mode === 'edit' + }, w(function (err, res) { + // Error while getting data? abort + if (err || !res || res.error) { + w.abort(); + return void noPadData(err || (!res ? 'EINVAL' : res.error)); + } + // No data found? abort + if (!Object.keys(res).length) { + w.abort(); + return void noPadData('NO_RESULT'); + } + // Data found but weaker? warn + if (parsed.hashData.mode === 'edit' && !res.href) { + return void badPadData(w(function (load) { + if (!load) { + w.abort(); + return; + } + newHref = res.roHref; + })); + } + // We have good data, keep the hash in memory + newHref = res.href; + })); + } + }).nThen(function (w) { + if (newHref) { + // Get the options (embed, present, etc.) of the hidden hash + // Use the same options in the full hash + var opts = parsed.getOptions(); + parsed = Utils.Hash.parsePadUrl(newHref); + currentPad.href = parsed.getUrl(opts); + currentPad.hash = parsed.hashData && parsed.hashData.getHash(opts); + } Cryptpad.getPadAttribute('title', w(function (err, data) { stored = (!err && typeof (data) === "string"); })); @@ -273,7 +334,7 @@ define([ if (parsed.type === "file") { // `isNewChannel` doesn't work for files (not a channel) // `getFileSize` is not adapted to channels because of metadata - Cryptpad.getFileSize(window.location.href, password, w(function (e, size) { + Cryptpad.getFileSize(currentPad.href, password, w(function (e, size) { if (size !== 0) { return void todo(); } // Wrong password or deleted file? askPassword(true, passwordCfg); @@ -281,7 +342,7 @@ define([ return; } // Not a file, so we can use `isNewChannel` - Cryptpad.isNewChannel(window.location.href, password, w(function(e, isNew) { + Cryptpad.isNewChannel(currentPad.href, password, w(function(e, isNew) { if (!isNew) { return void todo(); } if (parsed.hashData.mode === 'view' && (password || !parsed.hashData.password)) { // Error, wrong password stored, the view seed has changed with the password @@ -305,10 +366,12 @@ define([ } }).nThen(function (waitFor) { // Check if the pad exists on server - if (!window.location.hash) { isNewFile = true; return; } + if (!currentPad.hash) { isNewFile = true; return; } if (realtime) { - Cryptpad.isNewChannel(window.location.href, password, waitFor(function (e, isNew) { + // TODO we probably don't need to check again for password-protected pads + // (we use isNewChannel to test the password...) + Cryptpad.isNewChannel(currentPad.href, password, waitFor(function (e, isNew) { if (e) { return console.error(e); } isNewFile = Boolean(isNew); })); @@ -322,7 +385,7 @@ define([ readOnly = false; } Utils.crypto = Utils.Crypto.createEncryptor(Utils.secret.keys); - var parsed = Utils.Hash.parsePadUrl(window.location.href); + var parsed = Utils.Hash.parsePadUrl(currentPad.href); var burnAfterReading = parsed && parsed.hashData && parsed.hashData.ownerKey; if (!parsed.type) { throw new Error(); } var defaultTitle = Utils.UserObject.getDefaultName(parsed); @@ -342,7 +405,7 @@ define([ notifications = metaObj.user.notifications; })); if (typeof(isTemplate) === "undefined") { - Cryptpad.isTemplate(window.location.href, waitFor(function (err, t) { + Cryptpad.isTemplate(currentPad.href, waitFor(function (err, t) { if (err) { console.log(err); } isTemplate = t; })); @@ -368,7 +431,7 @@ define([ upgradeURL: Cryptpad.upgradeURL }, isNewFile: isNewFile, - isDeleted: isNewFile && window.location.hash.length > 0, + isDeleted: isNewFile && currentPad.hash.length > 0, forceCreationScreen: forceCreationScreen, password: password, channel: secret.channel, @@ -487,7 +550,7 @@ define([ }); sframeChan.on('Q_SET_LOGIN_REDIRECT', function (data, cb) { - sessionStorage.redirectTo = window.location.href; + sessionStorage.redirectTo = currentPad.href; cb(); }); @@ -570,7 +633,16 @@ define([ channel: secret.channel, path: initialPathInDrive // Where to store the pad if we don't have it in our drive }; - Cryptpad.setPadTitle(data, function (err) { + Cryptpad.setPadTitle(data, function (err, obj) { + if (!err && !(obj && obj.notStored)) { + // Pad is stored: hide the hash + var opts = parsed.getOptions(); + var hash = Utils.Hash.getHiddenHashFromKeys(parsed.type, secret, opts); + if (window.history && window.history.replaceState) { + if (!/^#/.test(hash)) { hash = '#' + hash; } + window.history.replaceState({}, window.document.title, hash); + } + } cb({error: err}); }); }); @@ -580,6 +652,9 @@ define([ }); sframeChan.on('EV_SET_HASH', function (hash) { + // In this case, we want to set the hash for the next page reload + // This hash is a category for the sidebar layout apps + // No need to store it in memory window.location.hash = hash; }); @@ -801,15 +876,19 @@ define([ // Present mode URL sframeChan.on('Q_PRESENT_URL_GET_VALUE', function (data, cb) { - var parsed = Utils.Hash.parsePadUrl(window.location.href); + var parsed = Utils.Hash.parsePadUrl(currentPad.href); cb(parsed.hashData && parsed.hashData.present); }); sframeChan.on('EV_PRESENT_URL_SET_VALUE', function (data) { - var parsed = Utils.Hash.parsePadUrl(window.location.href); - window.location.href = parsed.getUrl({ - embed: parsed.hashData.embed, - present: data - }); + // Update the rendered hash and the full hash with the "present" settings + var opts = parsed.getOptions(); + opts.present = data; + // Full hash + currentPad.href = parsed.getUrl(opts); + if (parsed.hashData) { currentPad.hash = parsed.hashData.getHash(opts); } + // Rendered (maybe hidden) hash + var hiddenParsed = Utils.Hash.parsePadUrl(window.location.href); + window.location.href = hiddenParsed.getUrl(opts); }); @@ -1011,7 +1090,7 @@ define([ }); sframeChan.on('Q_BLOB_PASSWORD_CHANGE', function (data, cb) { - data.href = data.href || window.location.href; + data.href = data.href || currentPad.href; var onPending = function (cb) { sframeChan.query('Q_BLOB_PASSWORD_CHANGE_PENDING', null, function (err, obj) { if (obj && obj.cancel) { cb(); } @@ -1027,12 +1106,12 @@ define([ }); sframeChan.on('Q_OO_PASSWORD_CHANGE', function (data, cb) { - data.href = data.href || window.location.href; + data.href = data.href || currentPad.href; Cryptpad.changeOOPassword(data, cb); }); sframeChan.on('Q_PAD_PASSWORD_CHANGE', function (data, cb) { - data.href = data.href || window.location.href; + data.href = data.href || currentPad.href; Cryptpad.changePadPassword(Cryptget, Crypto, data, cb); }); @@ -1234,7 +1313,11 @@ define([ var startRealtime = function (rtConfig) { rtConfig = rtConfig || {}; rtStarted = true; + var replaceHash = function (hash) { + // XXX Always put the full hash here. + // The pad has just been created but is not stored yet. We'll switch + // to hidden hash once the pad is stored if (window.history && window.history.replaceState) { if (!/^#/.test(hash)) { hash = '#' + hash; } window.history.replaceState({}, window.document.title, hash); @@ -1250,7 +1333,7 @@ define([ Cryptpad.padRpc.onReadyEvent.reg(function () { Cryptpad.burnPad({ password: password, - href: window.location.href, + href: currentPad.href, channel: secret.channel, ownerKey: burnAfterReading }); @@ -1265,7 +1348,7 @@ define([ readOnly: readOnly, crypto: Crypto.createEncryptor(secret.keys), onConnect: function () { - if (window.location.hash && window.location.hash !== '#') { + if (currentPad.hash && currentPad.hash !== '#') { /*window.location = parsed.getUrl({ present: parsed.hashData.present, embed: parsed.hashData.embed @@ -1278,11 +1361,11 @@ define([ }; nThen(function (waitFor) { - if (isNewFile && cfg.owned && !window.location.hash) { + if (isNewFile && cfg.owned && !currentPad.hash) { Cryptpad.getMetadata(waitFor(function (err, m) { cpNfCfg.owners = [m.priv.edPublic]; })); - } else if (isNewFile && !cfg.useCreationScreen && window.location.hash) { + } else if (isNewFile && !cfg.useCreationScreen && currentPad.hash) { console.log("new file with hash in the address bar in an app without pcs and which requires owners"); sframeChan.onReady(function () { sframeChan.query("EV_LOADING_ERROR", "DELETED"); @@ -1309,11 +1392,13 @@ define([ var ohc = window.onhashchange; window.onhashchange = function () {}; window.location.hash = newHash; + currentPad.hash = newHash; + currentPad.href = '/' + parsed.type + '/#' + newHash; window.onhashchange = ohc; ohc({reset: true}); // Update metadata values and send new metadata inside - parsed = Utils.Hash.parsePadUrl(window.location.href); + parsed = Utils.Hash.parsePadUrl(currentPad.href); defaultTitle = Utils.UserObject.getDefaultName(parsed); hashes = Utils.Hash.getHashes(secret); readOnly = false; diff --git a/www/poll/main.js b/www/poll/main.js index 2f62d3323..f2747b055 100644 --- a/www/poll/main.js +++ b/www/poll/main.js @@ -9,6 +9,7 @@ define([ var requireConfig = RequireConfig(); // Loaded in load #2 + var hash, href; nThen(function (waitFor) { DomReady.onReady(waitFor()); }).nThen(function (waitFor) { @@ -19,6 +20,14 @@ define([ }; window.rc = requireConfig; window.apiconf = ApiConfig; + + // Hidden hash + hash = window.location.hash; + href = window.location.href; + if (window.history && window.history.replaceState) { + window.history.replaceState({}, window.document.title, '#'); + } + document.getElementById('sbox-iframe').setAttribute('src', ApiConfig.httpSafeOrigin + '/poll/inner.html?' + requireConfig.urlArgs + '#' + encodeURIComponent(JSON.stringify(req))); @@ -37,6 +46,8 @@ define([ window.addEventListener('message', onMsg); }).nThen(function (/*waitFor*/) { SFCommonO.start({ + hash: hash, + href: href, useCreationScreen: true, messaging: true }); From a8e62505763f71dda0852b044240ffc17c04523a Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 27 Jan 2020 13:34:41 +0100 Subject: [PATCH 041/172] Hidden hash for shared folders and team invitation --- www/common/sframe-common-outer.js | 2 +- www/drive/main.js | 24 +++++++++++++++++++----- www/teams/main.js | 12 ++++++++++-- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 538fb4019..039bca5e7 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -144,7 +144,7 @@ define([ }); })); }).nThen(function (waitFor) { - if (!Utils.Hash.isValidHref(currentPad.href)) { + if (!Utils.Hash.isValidHref(window.location.href)) { waitFor.abort(); return void sframeChan.event('EV_LOADING_ERROR', 'INVALID_HASH'); } diff --git a/www/drive/main.js b/www/drive/main.js index bd2e506d4..0a7f78049 100644 --- a/www/drive/main.js +++ b/www/drive/main.js @@ -9,6 +9,7 @@ define([ var requireConfig = RequireConfig(); // Loaded in load #2 + var hash, href; nThen(function (waitFor) { DomReady.onReady(waitFor()); }).nThen(function (waitFor) { @@ -19,6 +20,14 @@ define([ }; window.rc = requireConfig; window.apiconf = ApiConfig; + + // Hidden hash + hash = window.location.hash; + href = window.location.href; + if (window.history && window.history.replaceState) { + window.history.replaceState({}, window.document.title, '#'); + } + document.getElementById('sbox-iframe').setAttribute('src', ApiConfig.httpSafeOrigin + '/drive/inner.html?' + requireConfig.urlArgs + '#' + encodeURIComponent(JSON.stringify(req))); @@ -37,19 +46,19 @@ define([ window.addEventListener('message', onMsg); }).nThen(function (/*waitFor*/) { var afterSecrets = function (Cryptpad, Utils, secret, cb) { - var hash = window.location.hash.slice(1); - if (hash && Utils.LocalStore.isLoggedIn()) { + var _hash = hash.slice(1); + if (_hash && Utils.LocalStore.isLoggedIn()) { // Add a shared folder! Cryptpad.addSharedFolder(null, secret, function (id) { window.CryptPad_newSharedFolder = id; cb(); }); return; - } else if (hash) { + } else if (_hash) { var id = Utils.Util.createRandomInteger(); window.CryptPad_newSharedFolder = id; var data = { - href: Utils.Hash.getRelativeHref(window.location.href), + href: Utils.Hash.getRelativeHref(href), password: secret.password }; return void Cryptpad.loadSharedFolder(id, data, cb); @@ -84,12 +93,15 @@ define([ }); sframeChan.on('EV_DRIVE_SET_HASH', function (hash) { // Update the hash in the address bar + // XXX Hidden hash: don't put the shared folder href in the address bar + /* if (!Utils.LocalStore.isLoggedIn()) { return; } var ohc = window.onhashchange; window.onhashchange = function () {}; window.location.hash = hash || ''; window.onhashchange = ohc; ohc({reset:true}); + */ }); Cryptpad.onNetworkDisconnect.reg(function () { sframeChan.event('EV_NETWORK_DISCONNECT'); @@ -109,9 +121,11 @@ define([ }; var addData = function (meta) { if (!window.CryptPad_newSharedFolder) { return; } - meta.anonSFHref = window.location.href; + meta.anonSFHref = href; }; SFCommonO.start({ + hash: hash, + href: href, afterSecrets: afterSecrets, noHash: true, noRealtime: true, diff --git a/www/teams/main.js b/www/teams/main.js index 559e90c7c..36c4e4f66 100644 --- a/www/teams/main.js +++ b/www/teams/main.js @@ -9,6 +9,7 @@ define([ var requireConfig = RequireConfig(); // Loaded in load #2 + var hash, href; nThen(function (waitFor) { DomReady.onReady(waitFor()); }).nThen(function (waitFor) { @@ -19,6 +20,14 @@ define([ }; window.rc = requireConfig; window.apiconf = ApiConfig; + + // Hidden hash + hash = window.location.hash; + href = window.location.href; + if (window.history && window.history.replaceState) { + window.history.replaceState({}, window.document.title, '#'); + } + document.getElementById('sbox-iframe').setAttribute('src', ApiConfig.httpSafeOrigin + '/teams/inner.html?' + requireConfig.urlArgs + '#' + encodeURIComponent(JSON.stringify(req))); @@ -37,7 +46,6 @@ define([ window.addEventListener('message', onMsg); }).nThen(function (/*waitFor*/) { var teamId; - var hash = window.location.hash.slice(1); var addRpc = function (sframeChan, Cryptpad) { sframeChan.on('Q_SET_TEAM', function (data, cb) { teamId = data; @@ -95,7 +103,7 @@ define([ }; var addData = function (meta) { if (!hash) { return; } - meta.teamInviteHash = hash; + meta.teamInviteHash = hash.slice(1); }; SFCommonO.start({ getSecrets: getSecrets, From 2c1e26cb527013256689db44f0e83cf5e64377bf Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 27 Jan 2020 13:37:52 +0100 Subject: [PATCH 042/172] Remove # symbol when no hash --- www/common/onlyoffice/main.js | 2 +- www/common/sframe-app-outer.js | 2 +- www/drive/main.js | 2 +- www/poll/main.js | 2 +- www/teams/main.js | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/www/common/onlyoffice/main.js b/www/common/onlyoffice/main.js index 600901872..7007bd360 100644 --- a/www/common/onlyoffice/main.js +++ b/www/common/onlyoffice/main.js @@ -24,7 +24,7 @@ define([ // Hidden hash hash = window.location.hash; href = window.location.href; - if (window.history && window.history.replaceState) { + if (window.history && window.history.replaceState && hash) { window.history.replaceState({}, window.document.title, '#'); } document.getElementById('sbox-iframe').setAttribute('src', diff --git a/www/common/sframe-app-outer.js b/www/common/sframe-app-outer.js index 563430c42..d85266ca7 100644 --- a/www/common/sframe-app-outer.js +++ b/www/common/sframe-app-outer.js @@ -23,7 +23,7 @@ define([ // Hidden hash hash = window.location.hash; href = window.location.href; - if (window.history && window.history.replaceState) { + if (window.history && window.history.replaceState && hash) { window.history.replaceState({}, window.document.title, '#'); } diff --git a/www/drive/main.js b/www/drive/main.js index 0a7f78049..00f64c2c3 100644 --- a/www/drive/main.js +++ b/www/drive/main.js @@ -24,7 +24,7 @@ define([ // Hidden hash hash = window.location.hash; href = window.location.href; - if (window.history && window.history.replaceState) { + if (window.history && window.history.replaceState && hash) { window.history.replaceState({}, window.document.title, '#'); } diff --git a/www/poll/main.js b/www/poll/main.js index f2747b055..d1bdca24d 100644 --- a/www/poll/main.js +++ b/www/poll/main.js @@ -24,7 +24,7 @@ define([ // Hidden hash hash = window.location.hash; href = window.location.href; - if (window.history && window.history.replaceState) { + if (window.history && window.history.replaceState && hash) { window.history.replaceState({}, window.document.title, '#'); } diff --git a/www/teams/main.js b/www/teams/main.js index 36c4e4f66..cfaae69a2 100644 --- a/www/teams/main.js +++ b/www/teams/main.js @@ -24,7 +24,7 @@ define([ // Hidden hash hash = window.location.hash; href = window.location.href; - if (window.history && window.history.replaceState) { + if (window.history && window.history.replaceState && hash) { window.history.replaceState({}, window.document.title, '#'); } From 7b9f86157e4decf1e2e39f92ce7991edc4b8e105 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 27 Jan 2020 13:53:54 +0100 Subject: [PATCH 043/172] Use version 3 for hidden hashes --- www/common/common-hash.js | 89 ++++++++++++++----------------- www/common/sframe-common-outer.js | 2 +- 2 files changed, 42 insertions(+), 49 deletions(-) diff --git a/www/common/common-hash.js b/www/common/common-hash.js index b92aea475..7869662d3 100644 --- a/www/common/common-hash.js +++ b/www/common/common-hash.js @@ -61,9 +61,9 @@ var factory = function (Util, Crypto, Nacl) { } }; Hash.getHiddenHashFromKeys = function (type, secret, opts) { - var mode = (secret.keys && secret.keys.editKeyStr) ? 'edit' : 'view'; + var mode = ((secret.keys && secret.keys.editKeyStr) || secret.key) ? 'edit' : 'view'; var pass = secret.password ? 'p/' : ''; - var hash = '/2/' + secret.type + '/' + mode + '/' + secret.channel + '/' + pass; + var hash = '/3/' + type + '/' + mode + '/' + secret.channel + '/' + pass; var href = '/' + type + '/#' + hash; var parsed = Hash.parsePadUrl(href); if (parsed.hashData && parsed.hashData.getHash) { @@ -172,12 +172,19 @@ Version 1 }; var parseTypeHash = Hash.parseTypeHash = function (type, hash) { if (!hash) { return; } - var options; + var options = []; var parsed = {}; var hashArr = fixDuplicateSlashes(hash).split('/'); if (['media', 'file', 'user', 'invite'].indexOf(type) === -1) { parsed.type = 'pad'; parsed.getHash = function () { return hash; }; + parsed.getOptions = function () { + return { + embed: parsed.embed, + present: parsed.present, + ownerKey: parsed.ownerKey + }; + }; if (hash.slice(0,1) !== '/' && hash.length >= 56) { // Version 0 // Old hash parsed.channel = hash.slice(0, 32); @@ -185,6 +192,24 @@ Version 1 parsed.version = 0; return parsed; } + + // Version >= 1: more hash options + parsed.getHash = function (opts) { + var hash = hashArr.slice(0, 5).join('/') + '/'; + var owner = typeof(opts.ownerKey) !== "undefined" ? opts.ownerKey : parsed.ownerKey; + if (owner) { hash += owner + '/'; } + if (parsed.password) { hash += 'p/'; } + if (opts.embed) { hash += 'embed/'; } + if (opts.present) { hash += 'present/'; } + return hash; + }; + var addOptions = function () { + parsed.password = options.indexOf('p') !== -1; + parsed.present = options.indexOf('present') !== -1; + parsed.embed = options.indexOf('embed') !== -1; + parsed.ownerKey = getOwnerKey(options); + }; + if (hashArr[1] && hashArr[1] === '1') { // Version 1 parsed.version = 1; parsed.mode = hashArr[2]; @@ -192,61 +217,30 @@ Version 1 parsed.key = Crypto.b64AddSlashes(hashArr[4]); options = hashArr.slice(5); - parsed.present = options.indexOf('present') !== -1; - parsed.embed = options.indexOf('embed') !== -1; - parsed.ownerKey = getOwnerKey(options); + addOptions(); - parsed.getHash = function (opts) { - var hash = hashArr.slice(0, 5).join('/') + '/'; - var owner = typeof(opts.ownerKey) !== "undefined" ? opts.ownerKey : parsed.ownerKey; - if (owner) { hash += owner + '/'; } - if (opts.embed) { hash += 'embed/'; } - if (opts.present) { hash += 'present/'; } - return hash; - }; - parsed.getOptions = function () { - return { - embed: parsed.embed, - present: parsed.present, - ownerKey: parsed.ownerKey - }; - }; return parsed; } if (hashArr[1] && hashArr[1] === '2') { // Version 2 parsed.version = 2; parsed.app = hashArr[2]; parsed.mode = hashArr[3]; + parsed.key = hashArr[4]; - // Check if the key is a channel ID - if (/^[a-f0-9]{32,34}$/.test(hashArr[4])) { - parsed.channel = hashArr[4]; - } else { - parsed.key = hashArr[4]; - } + options = hashArr.slice(5); + addOptions(); + + return parsed; + } + if (hashArr[1] && hashArr[1] === '3') { // Version 3: hidden hash + parsed.version = 3; + parsed.app = hashArr[2]; + parsed.mode = hashArr[3]; + parsed.channel = hashArr[4]; options = hashArr.slice(5); - parsed.password = options.indexOf('p') !== -1; - parsed.present = options.indexOf('present') !== -1; - parsed.embed = options.indexOf('embed') !== -1; - parsed.ownerKey = getOwnerKey(options); + addOptions(); - parsed.getHash = function (opts) { - var hash = hashArr.slice(0, 5).join('/') + '/'; - var owner = typeof(opts.ownerKey) !== "undefined" ? opts.ownerKey : parsed.ownerKey; - if (owner) { hash += owner + '/'; } - if (parsed.password) { hash += 'p/'; } - if (opts.embed) { hash += 'embed/'; } - if (opts.present) { hash += 'present/'; } - return hash; - }; - parsed.getOptions = function () { - return { - embed: parsed.embed, - present: parsed.present, - ownerKey: parsed.ownerKey - }; - }; return parsed; } return parsed; @@ -536,7 +530,6 @@ Version 1 if (parsed.hashData.type === 'pad' || parsed.hashData.type === 'file') { if (!parsed.hashData.key && !parsed.hashData.channel) { return; } if (parsed.hashData.key && !/^[a-zA-Z0-9+-/=]+$/.test(parsed.hashData.key)) { return; } - if (parsed.hashData.channel && !/^[a-f0-9]{32,34}$/.test(parsed.hashData.channel)) { return; } } } return true; diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 039bca5e7..a604cad38 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -30,7 +30,7 @@ define([ var password; var initialPathInDrive; - var currentPad = { + var currentPad = window.CryptPad_location = { href: cfg.href || window.location.href, hash: cfg.hash || window.location.hash }; From beaea7bb740ded17ccf4fde7b1f19900517bdbfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Benqu=C3=A9?= Date: Mon, 27 Jan 2020 14:05:15 +0000 Subject: [PATCH 044/172] change friends to contacts --- www/common/translations/messages.json | 36 +++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/www/common/translations/messages.json b/www/common/translations/messages.json index 62cb7a696..15dbe5184 100644 --- a/www/common/translations/messages.json +++ b/www/common/translations/messages.json @@ -282,7 +282,7 @@ "profile_description": "Description", "profile_fieldSaved": "New value saved: {0}", "profile_viewMyProfile": "View my profile", - "userlist_addAsFriendTitle": "Send \"{0}\" a friend request", + "userlist_addAsFriendTitle": "Send \"{0}\" a contact request", "contacts_title": "Contacts", "contacts_addError": "Error while adding that contact to the list", "contacts_added": "Contact invite accepted.", @@ -302,7 +302,7 @@ "contacts_confirmRemoveHistory": "Are you sure you want to permanently remove your chat history? Data cannot be restored", "contacts_removeHistoryServerError": "There was an error while removing your chat history. Try again later", "contacts_fetchHistory": "Retrieve older history", - "contacts_friends": "Friends", + "contacts_friends": "Contacts", "contacts_rooms": "Rooms", "contacts_leaveRoom": "Leave this room", "contacts_online": "Another user from this room is online", @@ -689,7 +689,7 @@ "features_f_social": "Social applications", "features_f_social_note": "Create a profile, use an avatar, chat with contacts", "features_f_file1": "Upload and share files", - "features_f_file1_note": "Share files with your friends or embed them in your pads", + "features_f_file1_note": "Share files with your contacts or embed them in your pads", "features_f_storage1": "Permanent storage (50MB)", "features_f_storage1_note": "Pads stored in your CryptDrive are never deleted for inactivity", "features_f_register": "Register for free", @@ -792,7 +792,7 @@ "a": "Registered users have access to a number of features unavailable to unregistered users. There's a chart here." }, "share": { - "q": "How can I share encrypted pads with my friends?", + "q": "How can I share encrypted pads with my contacts?", "a": "CryptPad puts the secret encryption key to your pad after the # character in the URL. Anything after this character is not sent to the server, so we never have access to your encryption keys. By sharing the link to a pad, you share the ability to read and access it." }, "remove": { @@ -824,7 +824,7 @@ "title": "Other questions", "pay": { "q": "Why should I pay when so many features are free?", - "a": "We give supporters additional storage and the ability to increase their friends' quotas (learn more).

Beyond these short term benefits, by subscribing with a premium account you help to fund continued, active development of CryptPad. That includes fixing bugs, adding new features, and making it easier for others to help host CryptPad themselves. Additionally, you help to prove to other service providers that people are willing to support privacy enhancing technologies. It is our hope that eventually business models based on selling user data will become a thing of the past.

Finally, we offer most of CryptPad's functionality for free because we believe everyone deserves personal privacy, not just those with disposable income. By supporting us, you help us continue to make it possible for underprivileged populations to access these basic features without a price tag attached." + "a": "We give supporters additional storage and the ability to increase their contacts' quotas (learn more).

Beyond these short term benefits, by subscribing with a premium account you help to fund continued, active development of CryptPad. That includes fixing bugs, adding new features, and making it easier for others to help host CryptPad themselves. Additionally, you help to prove to other service providers that people are willing to support privacy enhancing technologies. It is our hope that eventually business models based on selling user data will become a thing of the past.

Finally, we offer most of CryptPad's functionality for free because we believe everyone deserves personal privacy, not just those with disposable income. By supporting us, you help us continue to make it possible for underprivileged populations to access these basic features without a price tag attached." }, "goal": { "q": "What is your goal?", @@ -885,7 +885,7 @@ "colors": "Change the text and background colors using the and buttons" }, "poll": { - "decisions": "Make decisions in private among trusted friends", + "decisions": "Make decisions in private among trusted contacts", "options": "Propose options, and express your preferences", "choices": "Click cells in your column to cycle through yes (✔), maybe (~), or no (✖)", "submit": "Click submit to make your choices visible to others" @@ -903,7 +903,7 @@ }, "driveReadmeTitle": "What is CryptPad?", "readme_welcome": "Welcome to CryptPad !", - "readme_p1": "Welcome to CryptPad, this is where you can take note of things alone and with friends.", + "readme_p1": "Welcome to CryptPad, this is where you can take note of things alone and with contacts.", "readme_p2": "This pad will give you a quick walk through of how you can use CryptPad to take notes, keep them organized and work together on them.", "readme_cat1": "Get to know your CryptDrive", "readme_cat1_l1": "Make a pad: In your CryptDrive, click {0} then {1} and you can make a pad.", @@ -1075,17 +1075,17 @@ "friendRequest_later": "Decide later", "friendRequest_accept": "Accept (Enter)", "friendRequest_decline": "Decline", - "friendRequest_declined": "{0} declined your friend request", - "friendRequest_accepted": "{0} accepted your friend request", - "friendRequest_received": "{0} would like to be your friend", - "friendRequest_notification": "{0} sent you a friend request", + "friendRequest_declined": "{0} declined your contact request", + "friendRequest_accepted": "{0} accepted your contact request", + "friendRequest_received": "{0} would like to be your contact", + "friendRequest_notification": "{0} sent you a contact request", "notifications_empty": "No notifications available", "notifications_title": "You have unread notifications", "profile_addDescription": "Add a description", "profile_editDescription": "Edit your description", "profile_addLink": "Add a link to your website", "profile_info": "Other users can find your profile through your avatar in document user lists.", - "profile_friendRequestSent": "Friend request pending...", + "profile_friendRequestSent": "Contact request pending...", "profile_friend": "{0} is your friend", "notification_padShared": "{0} has shared a pad with you: {1}", "notification_fileShared": "{0} has shared a file with you: {1}", @@ -1097,7 +1097,7 @@ "share_withFriends": "Share", "notifications_dismiss": "Dismiss", "fm_info_sharedFolderHistory": "This is only the history of your shared folder: {0}
Your CryptDrive will stay in read-only mode while you navigate.", - "share_description": "Choose what you'd like to share and either get the link or send it directly to your CryptPad friends.", + "share_description": "Choose what you'd like to share and either get the link or send it directly to your CryptPad contacts.", "supportPage": "Support", "admin_cat_support": "Support", "admin_supportInitHelp": "Your server is not yet configured to have a support mailbox. If you want a support mailbox to receive messages from your users, you should ask your server administrator to run the script located in \"./scripts/generate-admin-keys.js\", then store the public key in the \"config.js\" file and send you the private key.", @@ -1130,7 +1130,7 @@ "notificationsPage": "Notifications", "openNotificationsApp": "Open notifications panel", "notifications_cat_all": "All", - "notifications_cat_friends": "Friend requests", + "notifications_cat_friends": "Contact requests", "notifications_cat_pads": "Shared with me", "notifications_cat_archived": "History", "notifications_dismissAll": "Dismiss all", @@ -1153,7 +1153,7 @@ "features_emailRequired": "Email address required", "owner_removeText": "Remove an existing owner", "owner_removePendingText": "Cancel a pending offer", - "owner_addText": "Offer co-ownership to a friend", + "owner_addText": "Offer co-ownership to a contact", "owner_unknownUser": "Unknown user", "owner_removeButton": "Remove selected owners", "owner_removePendingButton": "Cancel selected offers", @@ -1169,9 +1169,9 @@ "owner_removed": "{0} has removed your ownership of {1}", "owner_removedPending": "{0} has canceled your ownership offer for {1}", "share_linkTeam": "Add to team drive", - "team_pickFriends": "Choose which friends to invite to this team", + "team_pickFriends": "Choose which contacts to invite to this team", "team_inviteModalButton": "Invite", - "team_noFriend": "You haven't connected with any friends on CryptPad yet.", + "team_noFriend": "You haven't connected with any contacts on CryptPad yet.", "team_pcsSelectLabel": "Store in", "team_pcsSelectHelp": "Creating an owned pad in your team's drive will give ownership to the team.", "team_invitedToTeam": "{0} has invited you to join their team: {1}", @@ -1193,7 +1193,7 @@ "team_rosterPromote": "Promote", "team_rosterDemote": "Demote", "team_rosterKick": "Kick from the team", - "team_inviteButton": "Invite friends", + "team_inviteButton": "Invite contacts", "team_leaveButton": "Leave this team", "team_leaveConfirm": "If you leave this team you will lose access to its CryptDrive, chat history, and other contents. Are you sure?", "team_owner": "Owners", From cd586b626d640280e00aac5658e66a69840edc61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Benqu=C3=A9?= Date: Mon, 27 Jan 2020 14:06:57 +0000 Subject: [PATCH 045/172] unify keys that say 'X is/is not in your contacts' --- www/common/common-ui-elements.js | 4 ++-- www/common/translations/messages.json | 5 ++--- www/profile/inner.js | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index eb1169994..b5e5aa90a 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -4602,10 +4602,10 @@ define([ var f = priv.friends[curve]; $verified.append(h('span.fa.fa-certificate')); var $avatar = $(h('span.cp-avatar')).appendTo($verified); - $verified.append(h('p', Messages._getKey('requestEdit_fromFriend', [f.displayName]))); + $verified.append(h('p', Messages._getKey('isContact', [f.displayName]))); common.displayAvatar($avatar, f.avatar, f.displayName); } else { - $verified.append(Messages._getKey('requestEdit_fromStranger', [name])); + $verified.append(Messages._getKey('isNotContact', [name])); } return verified; }; diff --git a/www/common/translations/messages.json b/www/common/translations/messages.json index 15dbe5184..880edc12f 100644 --- a/www/common/translations/messages.json +++ b/www/common/translations/messages.json @@ -1086,7 +1086,8 @@ "profile_addLink": "Add a link to your website", "profile_info": "Other users can find your profile through your avatar in document user lists.", "profile_friendRequestSent": "Contact request pending...", - "profile_friend": "{0} is your friend", + "isContact": "{0} is in your contacts", + "isNotContact": "{0} is not in your contacts", "notification_padShared": "{0} has shared a pad with you: {1}", "notification_fileShared": "{0} has shared a file with you: {1}", "notification_folderShared": "{0} has shared a folder with you: {1}", @@ -1138,8 +1139,6 @@ "requestEdit_button": "Request edit rights", "requestEdit_dialog": "Are you sure you'd like to ask the owner of this pad for the ability to edit?", "requestEdit_confirm": "{1} has asked for the ability to edit the pad {0}. Would you like to grant them access?", - "requestEdit_fromFriend": "You are friends with {0}", - "requestEdit_fromStranger": "You are not friends with {0}", "requestEdit_viewPad": "Open the pad in a new tab", "later": "Decide later", "requestEdit_request": "{1} wants to edit the pad {0}", diff --git a/www/profile/inner.js b/www/profile/inner.js index d5243036c..0f4a17746 100644 --- a/www/profile/inner.js +++ b/www/profile/inner.js @@ -201,7 +201,7 @@ define([ // Add friend message APP.$friend.append(h('p.cp-app-profile-friend', [ h('i.fa.fa-address-book'), - Messages._getKey('profile_friend', [name]) + Messages._getKey('isContact', [name]) ])); if (!friends[data.curvePublic].notifications) { return; } // Add unfriend button From 7a02b074b734d7494ffe3ed3150a5c2c3290d764 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 27 Jan 2020 15:45:57 +0100 Subject: [PATCH 046/172] Hidden hash for files --- www/common/common-hash.js | 78 +++++++++++++++++++------------ www/common/cryptpad-common.js | 6 +-- www/common/outer/async-store.js | 3 +- www/common/sframe-common-outer.js | 3 +- www/file/main.js | 15 +++++- 5 files changed, 68 insertions(+), 37 deletions(-) diff --git a/www/common/common-hash.js b/www/common/common-hash.js index 7869662d3..7eb3ae6fa 100644 --- a/www/common/common-hash.js +++ b/www/common/common-hash.js @@ -60,10 +60,14 @@ var factory = function (Util, Crypto, Nacl) { return '/2/' + secret.type + '/view/' + Crypto.b64RemoveSlashes(data.viewKeyStr) + '/' + pass; } }; + Hash.getHiddenHashFromKeys = function (type, secret, opts) { - var mode = ((secret.keys && secret.keys.editKeyStr) || secret.key) ? 'edit' : 'view'; + var mode = ((secret.keys && secret.keys.editKeyStr) || secret.key) ? 'edit/' : 'view/'; var pass = secret.password ? 'p/' : ''; - var hash = '/3/' + type + '/' + mode + '/' + secret.channel + '/' + pass; + + if (secret.keys && secret.keys.fileKeyStr) { mode = ''; } + + var hash = '/3/' + type + '/' + mode + secret.channel + '/' + pass; var href = '/' + type + '/#' + hash; var parsed = Hash.parsePadUrl(href); if (parsed.hashData && parsed.hashData.getHash) { @@ -175,6 +179,14 @@ Version 1 var options = []; var parsed = {}; var hashArr = fixDuplicateSlashes(hash).split('/'); + + var addOptions = function () { + parsed.password = options.indexOf('p') !== -1; + parsed.present = options.indexOf('present') !== -1; + parsed.embed = options.indexOf('embed') !== -1; + parsed.ownerKey = getOwnerKey(options); + }; + if (['media', 'file', 'user', 'invite'].indexOf(type) === -1) { parsed.type = 'pad'; parsed.getHash = function () { return hash; }; @@ -203,12 +215,6 @@ Version 1 if (opts.present) { hash += 'present/'; } return hash; }; - var addOptions = function () { - parsed.password = options.indexOf('p') !== -1; - parsed.present = options.indexOf('present') !== -1; - parsed.embed = options.indexOf('embed') !== -1; - parsed.ownerKey = getOwnerKey(options); - }; if (hashArr[1] && hashArr[1] === '1') { // Version 1 parsed.version = 1; @@ -248,41 +254,53 @@ Version 1 parsed.getHash = function () { return hashArr.join('/'); }; if (['media', 'file'].indexOf(type) !== -1) { parsed.type = 'file'; + + parsed.getOptions = function () { + return { + embed: parsed.embed, + present: parsed.present, + ownerKey: parsed.ownerKey + }; + }; + + parsed.getHash = function (opts) { + var hash = hashArr.slice(0, 4).join('/') + '/'; + var owner = typeof(opts.ownerKey) !== "undefined" ? opts.ownerKey : parsed.ownerKey; + if (owner) { hash += owner + '/'; } + if (parsed.password) { hash += 'p/'; } + if (opts.embed) { hash += 'embed/'; } + if (opts.present) { hash += 'present/'; } + return hash; + }; + if (hashArr[1] && hashArr[1] === '1') { parsed.version = 1; parsed.channel = hashArr[2].replace(/-/g, '/'); parsed.key = hashArr[3].replace(/-/g, '/'); options = hashArr.slice(4); - parsed.ownerKey = getOwnerKey(options); + addOptions(); return parsed; } + if (hashArr[1] && hashArr[1] === '2') { // Version 2 parsed.version = 2; parsed.app = hashArr[2]; parsed.key = hashArr[3]; options = hashArr.slice(4); - parsed.password = options.indexOf('p') !== -1; - parsed.present = options.indexOf('present') !== -1; - parsed.embed = options.indexOf('embed') !== -1; - parsed.ownerKey = getOwnerKey(options); - - parsed.getHash = function (opts) { - var hash = hashArr.slice(0, 4).join('/') + '/'; - var owner = typeof(opts.ownerKey) !== "undefined" ? opts.ownerKey : parsed.ownerKey; - if (owner) { hash += owner + '/'; } - if (parsed.password) { hash += 'p/'; } - if (opts.embed) { hash += 'embed/'; } - if (opts.present) { hash += 'present/'; } - return hash; - }; - parsed.getOptions = function () { - return { - embed: parsed.embed, - present: parsed.present, - ownerKey: parsed.ownerKey - }; - }; + addOptions(); + + return parsed; + } + + if (hashArr[1] && hashArr[1] === '3') { // Version 3: hidden hash + parsed.version = 3; + parsed.app = hashArr[2]; + parsed.channel = hashArr[3]; + + options = hashArr.slice(4); + addOptions(); + return parsed; } return parsed; diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index 0be93e14c..1f612bc84 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -51,7 +51,7 @@ define([ // Store the href in memory // This is a placeholder value overriden in common.ready from sframe-common-outer - var currentPad = { + var currentPad = common.currentPad = { href: window.location.href }; @@ -763,7 +763,7 @@ define([ }; // Get data about a given channel: use with hidden hashes common.getPadDataFromChannel = function (obj, cb) { - if (!obj || !obj.channel || !obj.edit) { return void cb('EINVAL'); } + if (!obj || !obj.channel) { return void cb('EINVAL'); } postMessage("GET_PAD_DATA_FROM_CHANNEL", obj, function (data) { cb(void 0, data); }); @@ -1795,7 +1795,7 @@ define([ rdyCfg = rdyCfg || {}; if (rdyCfg.currentPad) { - currentPad = rdyCfg.currentPad; + currentPad = common.currentPad = rdyCfg.currentPad; } if (initialized) { diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index 4b2fda935..a06f593e7 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -1208,6 +1208,7 @@ define([ Store.getPadDataFromChannel = function (clientId, obj, cb) { var channel = obj.channel; var edit = obj.edit; + var isFile = obj.file; var res; var viewRes; getAllStores().some(function (s) { @@ -1217,7 +1218,7 @@ define([ if (!pad || !pad.data) { return; } var data = pad.data; // We've found a match: return the value and stop the loops - if ((edit && data.href) || (!edit && data.roHref)) { + if ((edit && data.href) || (!edit && data.roHref) || isFile) { res = data; return true; } diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index a604cad38..f9a2baa23 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -284,7 +284,8 @@ define([ if (!parsed.hashData.key && parsed.hashData.channel) { Cryptpad.getPadDataFromChannel({ channel: parsed.hashData.channel, - edit: parsed.hashData.mode === 'edit' + edit: parsed.hashData.mode === 'edit', + file: parsed.hashData.type === 'file' }, w(function (err, res) { // Error while getting data? abort if (err || !res || res.error) { diff --git a/www/file/main.js b/www/file/main.js index e59957299..8dc1c8eef 100644 --- a/www/file/main.js +++ b/www/file/main.js @@ -9,6 +9,7 @@ define([ var requireConfig = RequireConfig(); // Loaded in load #2 + var hash, href; nThen(function (waitFor) { DomReady.onReady(waitFor()); }).nThen(function (waitFor) { @@ -19,6 +20,14 @@ define([ }; window.rc = requireConfig; window.apiconf = ApiConfig; + + // Hidden hash + hash = window.location.hash; + href = window.location.href; + if (window.history && window.history.replaceState && hash) { + window.history.replaceState({}, window.document.title, '#'); + } + document.getElementById('sbox-iframe').setAttribute('src', ApiConfig.httpSafeOrigin + '/file/inner.html?' + requireConfig.urlArgs + '#' + encodeURIComponent(JSON.stringify(req))); @@ -36,10 +45,12 @@ define([ }; window.addEventListener('message', onMsg); }).nThen(function (/*waitFor*/) { - var addData = function (meta) { - meta.filehash = window.location.hash; + var addData = function (meta, Cryptpad) { + meta.filehash = Cryptpad.currentPad.hash; }; SFCommonO.start({ + hash: hash, + href: href, noRealtime: true, addData: addData }); From deddc8027071711fa59a36e045c7c5cb56e91cdf Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 27 Jan 2020 15:59:24 +0100 Subject: [PATCH 047/172] lnit compliance --- www/drive/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/drive/main.js b/www/drive/main.js index 00f64c2c3..64e990979 100644 --- a/www/drive/main.js +++ b/www/drive/main.js @@ -91,7 +91,7 @@ define([ cb(obj); }); }); - sframeChan.on('EV_DRIVE_SET_HASH', function (hash) { + sframeChan.on('EV_DRIVE_SET_HASH', function (/*hash*/) { // Update the hash in the address bar // XXX Hidden hash: don't put the shared folder href in the address bar /* From 83c35543b955f9abc039c524b3524f1c14e0fb54 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 27 Jan 2020 18:09:02 +0100 Subject: [PATCH 048/172] Keep the hash in the URL while the pad is loading --- www/common/sframe-common-outer.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index f9a2baa23..c92bc5121 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -142,6 +142,12 @@ define([ driveEvents: cfg.driveEvents, currentPad: currentPad }); + + if (window.history && window.history.replaceState && currentPad.hash) { + var nHash = currentPad.hash; + if (!/^#/.test(nHash)) { nHash = '#' + nHash; } + window.history.replaceState({}, window.document.title, nHash); + } })); }).nThen(function (waitFor) { if (!Utils.Hash.isValidHref(window.location.href)) { From 718610b6dbd869b0ea76f62c50d45482f111a266 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 27 Jan 2020 18:09:35 +0100 Subject: [PATCH 049/172] Use the hidden hash when opening a pad from the drive --- www/common/common-hash.js | 15 +++++++++++---- www/common/drive-ui.js | 28 ++++++++++++++-------------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/www/common/common-hash.js b/www/common/common-hash.js index 7eb3ae6fa..b9615ff1c 100644 --- a/www/common/common-hash.js +++ b/www/common/common-hash.js @@ -68,10 +68,9 @@ var factory = function (Util, Crypto, Nacl) { if (secret.keys && secret.keys.fileKeyStr) { mode = ''; } var hash = '/3/' + type + '/' + mode + secret.channel + '/' + pass; - var href = '/' + type + '/#' + hash; - var parsed = Hash.parsePadUrl(href); - if (parsed.hashData && parsed.hashData.getHash) { - return parsed.hashData.getHash(opts || {}); + var hashData = Hash.parseTypeHash(type, hash); + if (hashData && hashData.getHash) { + return hashData.getHash(opts || {}); } return hash; }; @@ -380,6 +379,14 @@ Version 1 return ret; }; + Hash.hashToHref = function (hash, type) { + return '/' + type + '/#' + hash; + }; + Hash.hrefToHref = function (href) { + var parsed = parsedPadUrl(href); + return parsed.hash; + }; + Hash.getRelativeHref = function (href) { if (!href) { return; } if (href.indexOf('#') === -1) { return; } diff --git a/www/common/drive-ui.js b/www/common/drive-ui.js index bc0fda97e..4abb98510 100644 --- a/www/common/drive-ui.js +++ b/www/common/drive-ui.js @@ -1029,15 +1029,18 @@ define([ return ret; }; - var openFile = function (el, href) { - if (!href) { - var data = manager.getFileData(el); - if (!data || (!data.href && !data.roHref)) { - return void logError("Missing data for the file", el, data); - } - href = data.href || data.roHref; + var openFile = function (el, isRo) { + var data = manager.getFileData(el); + if (!data || (!data.href && !data.roHref)) { + return void logError("Missing data for the file", el, data); } - window.open(APP.origin + href); + var href = data.href || data.roHref; + var parsed = Hash.parsePadUrl(href); + var secret = Hash.getSecrets(parsed.type, parsed.hash, data.password); + var hash = Hash.getHiddenHashFromKeys(parsed.type, secret); + var hiddenHref = Hash.hashToHref(hash, parsed.type); + // XXX hidden hash: use settings + window.open(APP.origin + hiddenHref); }; var refresh = APP.refresh = function () { @@ -3034,7 +3037,7 @@ define([ $icon.append(getFileIcon(r.id)); $type.text(Messages.type[parsed.type] || parsed.type); $title.click(function () { - openFile(null, r.data.href); + openFile(r.id); }); $atimeName.text(Messages.fm_lastAccess); $atime.text(new Date(r.data.atime).toLocaleString()); @@ -3944,15 +3947,12 @@ define([ // ANON_SHARED_FOLDER el = manager.find(paths[0].path.slice(1), APP.newSharedFolder); } - var href; if (manager.isPathIn(p.path, [FILES_DATA])) { - href = el.roHref; + el = p.path[1]; } else { if (!el || manager.isFolder(el)) { return; } - var data = manager.getFileData(el); - href = data.roHref; } - openFile(null, href); + openFile(el, true); }); } else if ($this.hasClass('cp-app-drive-context-openincode')) { From 50b897ee2e590ab506042e421052861d0ff55c24 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 27 Jan 2020 18:51:16 +0100 Subject: [PATCH 050/172] Hide the hash with autostore popup + fix anon shared folders --- www/common/sframe-common-outer.js | 13 ++++++++++++- www/drive/main.js | 12 +++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index c92bc5121..ad67db8ec 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -642,7 +642,8 @@ define([ }; Cryptpad.setPadTitle(data, function (err, obj) { if (!err && !(obj && obj.notStored)) { - // Pad is stored: hide the hash + // No error and the pad was correctly stored + // hide the hash var opts = parsed.getOptions(); var hash = Utils.Hash.getHiddenHashFromKeys(parsed.type, secret, opts); if (window.history && window.history.replaceState) { @@ -677,6 +678,16 @@ define([ forceSave: true }; Cryptpad.setPadTitle(data, function (err) { + if (!err && !(obj && obj.notStored)) { + // No error and the pad was correctly stored + // hide the hash + var opts = parsed.getOptions(); + var hash = Utils.Hash.getHiddenHashFromKeys(parsed.type, secret, opts); + if (window.history && window.history.replaceState) { + if (!/^#/.test(hash)) { hash = '#' + hash; } + window.history.replaceState({}, window.document.title, hash); + } + } cb({error: err}); }); }); diff --git a/www/drive/main.js b/www/drive/main.js index 64e990979..6e22e6b27 100644 --- a/www/drive/main.js +++ b/www/drive/main.js @@ -51,6 +51,12 @@ define([ // Add a shared folder! Cryptpad.addSharedFolder(null, secret, function (id) { window.CryptPad_newSharedFolder = id; + + // Clear the hash now that the secrets have been generated + if (window.history && window.history.replaceState && hash) { + window.history.replaceState({}, window.document.title, '#'); + } + cb(); }); return; @@ -58,7 +64,7 @@ define([ var id = Utils.Util.createRandomInteger(); window.CryptPad_newSharedFolder = id; var data = { - href: Utils.Hash.getRelativeHref(href), + href: Utils.Hash.getRelativeHref(Cryptpad.currentPad.href), password: secret.password }; return void Cryptpad.loadSharedFolder(id, data, cb); @@ -119,9 +125,9 @@ define([ sframeChan.event('EV_DRIVE_REMOVE', data); }); }; - var addData = function (meta) { + var addData = function (meta, Cryptpad) { if (!window.CryptPad_newSharedFolder) { return; } - meta.anonSFHref = href; + meta.anonSFHref = Cryptpad.currentPad.href; }; SFCommonO.start({ hash: hash, From ea65647d44d02974729b803e82ed22e4204627c2 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 27 Jan 2020 18:55:03 +0100 Subject: [PATCH 051/172] lint compliance --- www/common/common-hash.js | 2 +- www/common/drive-ui.js | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/www/common/common-hash.js b/www/common/common-hash.js index b9615ff1c..1d05710a8 100644 --- a/www/common/common-hash.js +++ b/www/common/common-hash.js @@ -383,7 +383,7 @@ Version 1 return '/' + type + '/#' + hash; }; Hash.hrefToHref = function (href) { - var parsed = parsedPadUrl(href); + var parsed = Hash.parsePadUrl(href); return parsed.hash; }; diff --git a/www/common/drive-ui.js b/www/common/drive-ui.js index 4abb98510..412c0be41 100644 --- a/www/common/drive-ui.js +++ b/www/common/drive-ui.js @@ -1037,6 +1037,10 @@ define([ var href = data.href || data.roHref; var parsed = Hash.parsePadUrl(href); var secret = Hash.getSecrets(parsed.type, parsed.hash, data.password); + if (isRo && secret.keys && secret.keys.editKeyStr) { + delete secret.keys.editKeyStr; + delete secret.key; + } var hash = Hash.getHiddenHashFromKeys(parsed.type, secret); var hiddenHref = Hash.hashToHref(hash, parsed.type); // XXX hidden hash: use settings From 80c012f34d018b32eebe21bd03cc721924840cad Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 27 Jan 2020 17:57:39 -0500 Subject: [PATCH 052/172] prepare to merge history keeper and rpc --- lib/api.js | 60 +++++++++++++++++++++++ lib/historyKeeper.js | 7 +-- server.js | 114 +++++++++++-------------------------------- 3 files changed, 91 insertions(+), 90 deletions(-) create mode 100644 lib/api.js diff --git a/lib/api.js b/lib/api.js new file mode 100644 index 000000000..60296d9b5 --- /dev/null +++ b/lib/api.js @@ -0,0 +1,60 @@ +/* jshint esversion: 6 */ +const nThen = require("nthen"); +const WebSocketServer = require('ws').Server; +const NetfluxSrv = require('chainpad-server/NetfluxWebsocketSrv'); + +module.exports.create = function (config) { + var historyKeeper; + var rpc; + const log = config.log; + const wsConfig = { + server: config.httpServer, + }; + + nThen(function (w) { + require('../storage/file').create(config, w(function (_store) { + config.store = _store; + })); + }).nThen(function (w) { + require("../storage/tasks").create(config, w(function (e, tasks) { + if (e) { + throw e; + } + config.tasks = tasks; + if (config.disableIntegratedTasks) { return; } + + // XXX support stopping this interval + setInterval(function () { + tasks.runAll(function (err) { + if (err) { + // either TASK_CONCURRENCY or an error with tasks.list + // in either case it is already logged. + } + }); + }, 1000 * 60 * 5); // run every five minutes + })); + }).nThen(function (w) { + require("./rpc").create(config, w(function (e, _rpc) { + if (e) { + w.abort(); + throw e; + } + rpc = _rpc; + })); + }).nThen(function () { + var HK = require('./historyKeeper.js'); + var hkConfig = { + tasks: config.tasks, + rpc: rpc, + store: config.store, + log: log, + }; + // XXX historyKeeper exports a `setConfig` method + historyKeeper = HK.create(hkConfig); + }).nThen(function () { + var wsSrv = new WebSocketServer(wsConfig); + // XXX NetfluxSrv shares some internal functions with historyKeeper + // by passing them to setConfig + NetfluxSrv.run(wsSrv, config, historyKeeper); + }); +}; diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index c3ed67cb7..55d6b1780 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -1,6 +1,5 @@ /* jshint esversion: 6 */ /* global Buffer */ -;(function () { 'use strict'; const nThen = require('nthen'); const Nacl = require('tweetnacl/nacl-fast'); @@ -63,6 +62,8 @@ const isValidValidateKeyString = function (key) { } }; +var CHECKPOINT_PATTERN = /^cp\|(([A-Za-z0-9+\/=]+)\|)?/; + module.exports.create = function (cfg) { const rpc = cfg.rpc; const tasks = cfg.tasks; @@ -385,8 +386,6 @@ module.exports.create = function (cfg) { return true; }; - var CHECKPOINT_PATTERN = /^cp\|(([A-Za-z0-9+\/=]+)\|)?/; - /* onChannelMessage Determine what we should store when a message a broadcasted to a channel" @@ -992,5 +991,3 @@ module.exports.create = function (cfg) { onDirectMessage: onDirectMessage, }; }; - -}()); diff --git a/server.js b/server.js index d2bdcabd8..70479d7ee 100644 --- a/server.js +++ b/server.js @@ -4,17 +4,12 @@ var Express = require('express'); var Http = require('http'); var Fs = require('fs'); -var WebSocketServer = require('ws').Server; -var NetfluxSrv = require('chainpad-server/NetfluxWebsocketSrv'); var Package = require('./package.json'); var Path = require("path"); var nThen = require("nthen"); var config = require("./lib/load-config"); -// support multiple storage back ends -var Storage = require('./storage/file'); - var app = Express(); // mode can be FRESH (default), DEV, or PACKAGE @@ -69,11 +64,9 @@ var setHeaders = (function () { if (Object.keys(headers).length) { return function (req, res) { const h = [ - /^\/pad(2)?\/inner\.html.*/, + /^\/pad\/inner\.html.*/, /^\/common\/onlyoffice\/.*\/index\.html.*/, - /^\/sheet\/inner\.html.*/, - /^\/ooslide\/inner\.html.*/, - /^\/oodoc\/inner\.html.*/, + /^\/(sheet|ooslide|oodoc)\/inner\.html.*/, ].some((regex) => { return regex.test(req.url) }) ? padHeaders : headers; @@ -117,11 +110,6 @@ app.use(function (req, res, next) { app.use(Express.static(__dirname + '/www')); -Fs.exists(__dirname + "/customize", function (e) { - if (e) { return; } - console.log("Cryptpad is customizable, see customize.dist/readme.md for details"); -}); - // FIXME I think this is a regression caused by a recent PR // correct this hack without breaking the contributor's intended behaviour. @@ -207,80 +195,36 @@ app.use(function (req, res, next) { var httpServer = Http.createServer(app); -httpServer.listen(config.httpPort,config.httpAddress,function(){ - var host = config.httpAddress; - var hostName = !host.indexOf(':') ? '[' + host + ']' : host; - - var port = config.httpPort; - var ps = port === 80? '': ':' + port; - - console.log('[%s] server available http://%s%s', new Date().toISOString(), hostName, ps); -}); -if (config.httpSafePort) { - Http.createServer(app).listen(config.httpSafePort, config.httpAddress); -} - -var wsConfig = { server: httpServer }; +nThen(function (w) { + Fs.exists(__dirname + "/customize", w(function (e) { + if (e) { return; } + console.log("Cryptpad is customizable, see customize.dist/readme.md for details"); + })); +}).nThen(function (w) { + httpServer.listen(config.httpPort,config.httpAddress,function(){ + var host = config.httpAddress; + var hostName = !host.indexOf(':') ? '[' + host + ']' : host; -var rpc; -var historyKeeper; + var port = config.httpPort; + var ps = port === 80? '': ':' + port; -var log; + console.log('[%s] server available http://%s%s', new Date().toISOString(), hostName, ps); + }); -// Initialize logging, the the store, then tasks, then rpc, then history keeper and then start the server -var nt = nThen(function (w) { - // set up logger - var Logger = require("./lib/log"); - //console.log("Loading logging module"); - Logger.create(config, w(function (_log) { - log = config.log = _log; - })); -}).nThen(function (w) { - if (config.externalWebsocketURL) { - // if you plan to use an external websocket server - // then you don't need to load any API services other than the logger. - // Just abort. - w.abort(); - return; + if (config.httpSafePort) { + Http.createServer(app).listen(config.httpSafePort, config.httpAddress, w()); } - Storage.create(config, w(function (_store) { - config.store = _store; - })); -}).nThen(function (w) { - var Tasks = require("./storage/tasks"); - Tasks.create(config, w(function (e, tasks) { - if (e) { - throw e; - } - config.tasks = tasks; - if (config.disableIntegratedTasks) { return; } - setInterval(function () { - tasks.runAll(function (err) { - if (err) { - // either TASK_CONCURRENCY or an error with tasks.list - // in either case it is already logged. - } - }); - }, 1000 * 60 * 5); // run every five minutes - })); -}).nThen(function (w) { - require("./lib/rpc").create(config, w(function (e, _rpc) { - if (e) { - w.abort(); - throw e; - } - rpc = _rpc; - })); }).nThen(function () { - var HK = require('./lib/historyKeeper.js'); - var hkConfig = { - tasks: config.tasks, - rpc: rpc, - store: config.store, - log: log, - }; - historyKeeper = HK.create(hkConfig); -}).nThen(function () { - var wsSrv = new WebSocketServer(wsConfig); - NetfluxSrv.run(wsSrv, config, historyKeeper); + var wsConfig = { server: httpServer }; + + // Initialize logging then start the API server + require("./lib/log").create(config, function (_log) { + config.log = _log; + config.httpServer = httpServer; + + if (config.externalWebsocketURL) { return; } + require("./lib/api").create(config); + }); }); + + From b922860339e92f5753edbfba428228f692261914 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 27 Jan 2020 18:54:16 -0500 Subject: [PATCH 053/172] drop usage of historyKeeper.setConfig --- lib/api.js | 7 ++--- lib/historyKeeper.js | 69 ++++++++++++++++++++++++-------------------- lib/hk-util.js | 9 ++++++ 3 files changed, 49 insertions(+), 36 deletions(-) diff --git a/lib/api.js b/lib/api.js index 60296d9b5..d633db59a 100644 --- a/lib/api.js +++ b/lib/api.js @@ -4,7 +4,6 @@ const WebSocketServer = require('ws').Server; const NetfluxSrv = require('chainpad-server/NetfluxWebsocketSrv'); module.exports.create = function (config) { - var historyKeeper; var rpc; const log = config.log; const wsConfig = { @@ -50,11 +49,9 @@ module.exports.create = function (config) { log: log, }; // XXX historyKeeper exports a `setConfig` method - historyKeeper = HK.create(hkConfig); - }).nThen(function () { + var wsSrv = new WebSocketServer(wsConfig); - // XXX NetfluxSrv shares some internal functions with historyKeeper - // by passing them to setConfig + var historyKeeper = HK.create(hkConfig); NetfluxSrv.run(wsSrv, config, historyKeeper); }); }; diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 55d6b1780..f8d3c8a15 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -10,6 +10,8 @@ const WriteQueue = require("./write-queue"); const BatchRead = require("./batch-read"); const Extras = require("./hk-util.js"); +const STANDARD_CHANNEL_LENGTH = Extras.STANDARD_CHANNEL_LENGTH; +const EPHEMERAL_CHANNEL_LENGTH = Extras.EPHEMERAL_CHANNEL_LENGTH; let Log; const now = function () { return (new Date()).getTime(); }; @@ -77,14 +79,6 @@ module.exports.create = function (cfg) { Log.verbose('HK_ID', 'History keeper ID: ' + HISTORY_KEEPER_ID); - let sendMsg = function () {}; - let STANDARD_CHANNEL_LENGTH, EPHEMERAL_CHANNEL_LENGTH; - const setConfig = function (config) { - STANDARD_CHANNEL_LENGTH = config.STANDARD_CHANNEL_LENGTH; - EPHEMERAL_CHANNEL_LENGTH = config.EPHEMERAL_CHANNEL_LENGTH; - sendMsg = config.sendMsg; - }; - /* computeIndex can call back with an error or a computed index which includes: * cpIndex: @@ -326,7 +320,7 @@ module.exports.create = function (cfg) { const historyKeeperBroadcast = function (ctx, channel, msg) { let chan = ctx.channels[channel] || (([] /*:any*/) /*:Chan_t*/); chan.forEach(function (user) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)]); }); }; @@ -707,7 +701,7 @@ module.exports.create = function (cfg) { // parsed[1] is the channel id // parsed[2] is a validation key or an object containing metadata (optionnal) // parsed[3] is the last known hash (optionnal) - sendMsg(ctx, user, [seq, 'ACK']); + ctx.sendMsg(ctx, user, [seq, 'ACK']); var channelName = parsed[1]; var config = parsed[2]; var metadata = {}; @@ -758,7 +752,7 @@ module.exports.create = function (cfg) { // FIXME this is hard to read because 'checkExpired' has side effects if (checkExpired(ctx, channelName)) { return void waitFor.abort(); } // always send metadata with GET_HISTORY requests - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); })); }).nThen(() => { let msgCount = 0; @@ -769,12 +763,12 @@ module.exports.create = function (cfg) { msgCount++; // avoid sending the metadata message a second time if (isMetadataMessage(msg) && metadata_cache[channelName]) { return readMore(); } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)], readMore); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)], readMore); }, (err) => { if (err && err.code !== 'ENOENT') { if (err.message !== 'EINVAL') { Log.error("HK_GET_HISTORY", err); } const parsedMsg = {error:err.message, channel: channelName}; - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); return; } @@ -817,12 +811,12 @@ module.exports.create = function (cfg) { } }); } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(metadata)]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(metadata)]); } // End of history message: let parsedMsg = {state: 1, channel: channelName}; - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); }); }); }; @@ -831,7 +825,7 @@ module.exports.create = function (cfg) { var channelName = parsed[1]; var map = parsed[2]; if (!(map && typeof(map) === 'object')) { - return void sendMsg(ctx, user, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]); + return void ctx.sendMsg(ctx, user, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]); } var oldestKnownHash = map.from; @@ -839,14 +833,14 @@ module.exports.create = function (cfg) { var desiredCheckpoint = map.cpCount; var txid = map.txid; if (typeof(desiredMessages) !== 'number' && typeof(desiredCheckpoint) !== 'number') { - return void sendMsg(ctx, user, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]); + return void ctx.sendMsg(ctx, user, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]); } if (!txid) { - return void sendMsg(ctx, user, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]); + return void ctx.sendMsg(ctx, user, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]); } - sendMsg(ctx, user, [seq, 'ACK']); + ctx.sendMsg(ctx, user, [seq, 'ACK']); return void getOlderHistory(channelName, oldestKnownHash, function (messages) { var toSend = []; if (typeof (desiredMessages) === "number") { @@ -862,11 +856,11 @@ module.exports.create = function (cfg) { } } toSend.forEach(function (msg) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['HISTORY_RANGE', txid, msg])]); }); - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['HISTORY_RANGE_END', txid, channelName]) ]); }); @@ -876,20 +870,20 @@ module.exports.create = function (cfg) { // parsed[1] is the channel id // parsed[2] is a validation key (optionnal) // parsed[3] is the last known hash (optionnal) - sendMsg(ctx, user, [seq, 'ACK']); + ctx.sendMsg(ctx, user, [seq, 'ACK']); // FIXME should we send metadata here too? // none of the clientside code which uses this API needs metadata, but it won't hurt to send it (2019-08-22) return void getHistoryAsync(ctx, parsed[1], -1, false, (msg, readMore) => { if (!msg) { return; } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['FULL_HISTORY', msg])], readMore); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['FULL_HISTORY', msg])], readMore); }, (err) => { let parsedMsg = ['FULL_HISTORY_END', parsed[1]]; if (err) { Log.error('HK_GET_FULL_HISTORY', err.stack); parsedMsg = ['ERROR', parsed[1], err.message]; } - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]); }); }; @@ -899,12 +893,12 @@ module.exports.create = function (cfg) { /* RPC Calls... */ var rpc_call = parsed.slice(1); - sendMsg(ctx, user, [seq, 'ACK']); + ctx.sendMsg(ctx, user, [seq, 'ACK']); try { // slice off the sequence number and pass in the rest of the message rpc(ctx, rpc_call, function (err, output) { if (err) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', err])]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', err])]); return; } var msg = rpc_call[0].slice(); @@ -930,7 +924,7 @@ module.exports.create = function (cfg) { let chan = ctx.channels[output.channel]; if (chan && chan.length) { chan.forEach(function (user) { - sendMsg(ctx, user, output.message); + ctx.sendMsg(ctx, user, output.message); //[0, null, 'MSG', user.id, JSON.stringify(output.message)]); }); } @@ -940,10 +934,10 @@ module.exports.create = function (cfg) { } // finally, send a response to the client that sent the RPC - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0]].concat(output))]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0]].concat(output))]); }); } catch (e) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]); + ctx.sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]); } }; @@ -982,12 +976,25 @@ module.exports.create = function (cfg) { command(ctx, seq, user, parsed); }; + // XXX every one of these values is exported because + // netfluxWebsocketServer needs them to do some magic historyKeeper things + // we could have netflux emit events and let historyKeeper handle them instead return { id: HISTORY_KEEPER_ID, - setConfig: setConfig, - onChannelMessage: onChannelMessage, + + // XXX dropChannel allows netflux to clear historyKeeper's cache + // maybe it should emit a 'channel_dropped' event instead + // and let historyKeeper decide what to do dropChannel: dropChannel, + + // XXX we don't need to export checkExpired if netflux allows it to be HK's responsibility checkExpired: checkExpired, + + // XXX again, if netflux emitted events then historyKeeper could handle them itself + // and netflux wouldn't need to have historyKeeper-specific code onDirectMessage: onDirectMessage, + + // XXX same + onChannelMessage: onChannelMessage, }; }; diff --git a/lib/hk-util.js b/lib/hk-util.js index cea39c4e1..aaa861054 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -22,3 +22,12 @@ HK.getHash = function (msg, Log) { return msg.slice(0,64); }; +// historyKeeper should explicitly store any channel +// with a 32 character id +HK.STANDARD_CHANNEL_LENGTH = 32; + +// historyKeeper should not store messages sent to any channel +// with a 34 character id +HK.EPHEMERAL_CHANNEL_LENGTH = 34; + + From 0237bb2867a056d1686f4f9c5993a53688bea6f2 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 28 Jan 2020 10:46:26 +0100 Subject: [PATCH 054/172] Fix read-only pads --- www/common/sframe-common-outer.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index ad67db8ec..0197d68ee 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -288,9 +288,10 @@ define([ var newHref; nThen(function (w) { if (!parsed.hashData.key && parsed.hashData.channel) { + var edit = parsed.hashData.mode === 'edit'; Cryptpad.getPadDataFromChannel({ channel: parsed.hashData.channel, - edit: parsed.hashData.mode === 'edit', + edit: edit, file: parsed.hashData.type === 'file' }, w(function (err, res) { // Error while getting data? abort @@ -304,7 +305,7 @@ define([ return void noPadData('NO_RESULT'); } // Data found but weaker? warn - if (parsed.hashData.mode === 'edit' && !res.href) { + if (edit && !res.href) { return void badPadData(w(function (load) { if (!load) { w.abort(); @@ -314,7 +315,7 @@ define([ })); } // We have good data, keep the hash in memory - newHref = res.href; + newHref = edit ? res.href : (res.roHref || res.href); })); } }).nThen(function (w) { From 6183401a6f810bd62d9defc4282f1e6ef28ab7f5 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 28 Jan 2020 11:31:03 +0100 Subject: [PATCH 055/172] Add settings to continue using unsafe links --- www/common/common-interface.js | 31 +++++++++++++++++ www/common/drive-ui.js | 10 ++++-- www/common/sframe-common-outer.js | 8 +++-- www/settings/inner.js | 57 +++++++++++++++++++++++++++++-- 4 files changed, 99 insertions(+), 7 deletions(-) diff --git a/www/common/common-interface.js b/www/common/common-interface.js index 9cc1c1efb..ba5ca378b 100644 --- a/www/common/common-interface.js +++ b/www/common/common-interface.js @@ -1104,5 +1104,36 @@ define([ }; }; + UI.makeSpinner = function ($container) { + var $ok = $('', {'class': 'fa fa-check', title: Messages.saved}).hide(); + var $spinner = $('', {'class': 'fa fa-spinner fa-pulse'}).hide(); + + var spin = function () { + $ok.hide(); + $spinner.show(); + }; + var hide = function () { + $ok.hide(); + $spinner.hide(); + }; + var done = function () { + $ok.show(); + $spinner.hide(); + }; + + if ($container && $container.append) { + $container.append($ok); + $container.append($spinner); + } + + return { + ok: $ok[0], + spinner: $spinner[0], + spin: spin, + hide: hide, + done: done + } + }; + return UI; }); diff --git a/www/common/drive-ui.js b/www/common/drive-ui.js index 412c0be41..b0396773c 100644 --- a/www/common/drive-ui.js +++ b/www/common/drive-ui.js @@ -1034,7 +1034,14 @@ define([ if (!data || (!data.href && !data.roHref)) { return void logError("Missing data for the file", el, data); } - var href = data.href || data.roHref; + var href = isRo ? data.roHref : (data.href || data.roHref); + var priv = metadataMgr.getPrivateData(); + var useUnsafe = Util.find(priv, ['settings', 'security', 'unsafeLinks']); + if (useUnsafe) { + return void window.open(APP.origin + href); + } + + // Get hidden hash var parsed = Hash.parsePadUrl(href); var secret = Hash.getSecrets(parsed.type, parsed.hash, data.password); if (isRo && secret.keys && secret.keys.editKeyStr) { @@ -1043,7 +1050,6 @@ define([ } var hash = Hash.getHiddenHashFromKeys(parsed.type, secret); var hiddenHref = Hash.hashToHref(hash, parsed.type); - // XXX hidden hash: use settings window.open(APP.origin + hiddenHref); }; diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 0197d68ee..e2472928e 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -398,6 +398,7 @@ define([ if (!parsed.type) { throw new Error(); } var defaultTitle = Utils.UserObject.getDefaultName(parsed); var edPublic, curvePublic, notifications, isTemplate; + var settings = {}; var forceCreationScreen = cfg.useCreationScreen && sessionStorage[Utils.Constants.displayPadCreationScreen]; delete sessionStorage[Utils.Constants.displayPadCreationScreen]; @@ -411,6 +412,7 @@ define([ edPublic = metaObj.priv.edPublic; // needed to create an owned pad curvePublic = metaObj.user.curvePublic; notifications = metaObj.user.notifications; + settings = metaObj.priv.settings; })); if (typeof(isTemplate) === "undefined") { Cryptpad.isTemplate(currentPad.href, waitFor(function (err, t) { @@ -647,7 +649,8 @@ define([ // hide the hash var opts = parsed.getOptions(); var hash = Utils.Hash.getHiddenHashFromKeys(parsed.type, secret, opts); - if (window.history && window.history.replaceState) { + var useUnsafe = Utils.Util.find(settings, ['security', 'unsafeLinks']); + if (!useUnsafe && window.history && window.history.replaceState) { if (!/^#/.test(hash)) { hash = '#' + hash; } window.history.replaceState({}, window.document.title, hash); } @@ -684,7 +687,8 @@ define([ // hide the hash var opts = parsed.getOptions(); var hash = Utils.Hash.getHiddenHashFromKeys(parsed.type, secret, opts); - if (window.history && window.history.replaceState) { + var useUnsafe = Utils.Util.find(settings, ['security', 'unsafeLinks']); + if (!useUnsafe && window.history && window.history.replaceState) { if (!/^#/.test(hash)) { hash = '#' + hash; } window.history.replaceState({}, window.document.title, hash); } diff --git a/www/settings/inner.js b/www/settings/inner.js index 7c9dd0630..b3aa9bb0b 100644 --- a/www/settings/inner.js +++ b/www/settings/inner.js @@ -53,13 +53,15 @@ define([ 'cp-settings-language-selector', 'cp-settings-resettips', 'cp-settings-logout-everywhere', - 'cp-settings-autostore', 'cp-settings-userfeedback', 'cp-settings-change-password', 'cp-settings-migrate', - 'cp-settings-backup', 'cp-settings-delete' ], + 'security': [ // XXX + 'cp-settings-autostore', + 'cp-settings-safe-links', + ], 'creation': [ 'cp-settings-creation-owned', 'cp-settings-creation-expire', @@ -115,6 +117,24 @@ define([ var create = {}; + var makeBlock = function (key, getter, full) { + var safeKey = key.replace(/-([a-z])/g, function (g) { return g[1].toUpperCase(); }); + + create[key] = function () { + var $div = $('
', {'class': 'cp-settings-' + key + ' cp-sidebarlayout-element'}); + if (full) { + $('