From 9ce7cea9ccdecb7cbfd4afc6d0704a59173dfa31 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 4 Sep 2019 17:22:06 +0200 Subject: [PATCH 01/18] abstract historyKeeper's batched reads implementation into a library --- historyKeeper.js | 45 ++++++++---------------------------- lib/batch-read.js | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 35 deletions(-) create mode 100644 lib/batch-read.js diff --git a/historyKeeper.js b/historyKeeper.js index 07a555e5f..b84bb8bc3 100644 --- a/historyKeeper.js +++ b/historyKeeper.js @@ -8,6 +8,7 @@ const Crypto = require('crypto'); const Once = require("./lib/once"); const Meta = require("./lib/metadata"); const WriteQueue = require("./lib/write-queue"); +const BatchRead = require("./lib/batch-read"); let Log; const now = function () { return (new Date()).getTime(); }; @@ -231,7 +232,7 @@ module.exports.create = function (cfg) { as an added bonus: if the channel exists but its index does not then it caches the index */ - const indexQueues = {}; + const batchIndexReads = BatchRead(); const getIndex = (ctx, channelName, cb) => { const chan = ctx.channels[channelName]; // if there is a channel in memory and it has an index cached, return it @@ -242,40 +243,14 @@ module.exports.create = function (cfg) { }); } - // if a call to computeIndex is already in progress for this channel - // then add the callback for the latest invocation to the queue - // and wait for it to complete - if (Array.isArray(indexQueues[channelName])) { - indexQueues[channelName].push(cb); - return; - } - - // otherwise, make a queue for any 'getIndex' calls made before the following 'computeIndex' call completes - var queue = indexQueues[channelName] = (indexQueues[channelName] || [cb]); - - computeIndex(channelName, (err, ret) => { - if (!Array.isArray(queue)) { - // something is very wrong if there's no callback array - return void Log.error("E_INDEX_NO_CALLBACK", channelName); - } - - - // clean up the queue that you're about to handle, but keep a local copy - delete indexQueues[channelName]; - - // this is most likely an unrecoverable filesystem error - if (err) { - // call back every pending function with the error - return void queue.forEach(function (_cb) { - _cb(err); - }); - } - // cache the computed result if possible - if (chan) { chan.index = ret; } - - // call back every pending function with the result - queue.forEach(function (_cb) { - _cb(void 0, ret); + batchIndexReads(channelName, cb, function (done) { + computeIndex(channelName, (err, ret) => { + // this is most likely an unrecoverable filesystem error + if (err) { return void done(err); } + // cache the computed result if possible + if (chan) { chan.index = ret; } + // return + done(void 0, ret); }); }); }; diff --git a/lib/batch-read.js b/lib/batch-read.js new file mode 100644 index 000000000..66f106c54 --- /dev/null +++ b/lib/batch-read.js @@ -0,0 +1,58 @@ +/* + +## Purpose + +To avoid running expensive IO or computation concurrently. + +If the result of IO or computation is requested while an identical request +is already in progress, wait until the first one completes and provide its +result to every routine that requested it. + +## Usage + +Provide: + +1. a named key for the computation or resource, +2. a callback to handle the result +3. an implementation which calls back with the result + +``` +var batch = Batch(); + +var read = function (path, cb) { + batch(path, cb, function (done) { + console.log("reading %s", path); + fs.readFile(path, 'utf8', done); + }); +}; + +read('./pewpew.txt', function (err, data) { + if (err) { return void console.error(err); } + console.log(data); +}); + +read('./pewpew.txt', function (err, data) { + if (err) { return void console.error(err); } + console.log(data); +}); +``` + +*/ + +module.exports = function () { + var map = {}; + return function (id, cb, impl) { + if (typeof(cb) !== 'function' || typeof(impl) !== 'function') { + throw new Error("expected callback and implementation"); + } + if (map[id]) { return void map[id].push(cb); } + map[id] = [cb]; + impl(function () { + var args = Array.prototype.slice.call(arguments); + map[id].forEach(function (h) { + h.apply(null, args); + }); + delete map[id]; + }); + }; +}; From 096b0fb44aba3687b45068cfedf58114de229824 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 5 Sep 2019 12:22:30 +0200 Subject: [PATCH 02/18] move 'getDefaultName' out of 'common-hash' --- www/common/common-hash.js | 17 +---------------- www/common/cryptpad-common.js | 1 - www/common/outer/async-store.js | 4 +++- www/common/outer/userObject.js | 2 +- www/common/sframe-common-outer.js | 8 +++++--- www/common/userObject.js | 16 ++++++++++++++++ 6 files changed, 26 insertions(+), 22 deletions(-) diff --git a/www/common/common-hash.js b/www/common/common-hash.js index e49b0e217..756877857 100644 --- a/www/common/common-hash.js +++ b/www/common/common-hash.js @@ -1,9 +1,8 @@ define([ '/common/common-util.js', - '/customize/messages.js', '/bower_components/chainpad-crypto/crypto.js', '/bower_components/tweetnacl/nacl-fast.min.js' -], function (Util, Messages, Crypto) { +], function (Util, Crypto) { var Nacl = window.nacl; var Hash = window.CryptPad_Hash = {}; @@ -515,20 +514,6 @@ Version 1 '/' + curvePublic.replace(/\//g, '-') + '/'; }; - // Create untitled documents when no name is given - var getLocaleDate = function () { - if (window.Intl && window.Intl.DateTimeFormat) { - var options = {weekday: "short", year: "numeric", month: "long", day: "numeric"}; - return new window.Intl.DateTimeFormat(undefined, options).format(new Date()); - } - return new Date().toString().split(' ').slice(0,4).join(' '); - }; - Hash.getDefaultName = function (parsed) { - var type = parsed.type; - var name = (Messages.type)[type] + ' - ' + getLocaleDate(); - return name; - }; - Hash.isValidHref = function (href) { // Non-empty href? if (!href) { return; } diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index f8c3f1545..0c4ccb3a3 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -648,7 +648,6 @@ define([ data.href = parsed.getUrl({present: parsed.present}); if (typeof (data.title) !== "string") { return cb('Missing title'); } - if (data.title.trim() === "") { data.title = Hash.getDefaultName(parsed); } if (common.initialPath) { if (!data.path) { diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index 4a7696749..928978574 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -507,7 +507,7 @@ define([ roHref: roHref, atime: now, ctime: now, - title: title || Hash.getDefaultName(Hash.parsePadUrl(href)), + title: title || UserObject.getDefaultName(Hash.parsePadUrl(href)), }; }; @@ -805,6 +805,8 @@ define([ var p = Hash.parsePadUrl(href); var h = p.hashData; + if (title.trim() === "") { title = UserObject.getDefaultName(p); } + if (AppConfig.disableAnonymousStore && !store.loggedIn) { return void cb(); } if (p.type === "debug") { return void cb(); } diff --git a/www/common/outer/userObject.js b/www/common/outer/userObject.js index 4f235366a..64673c882 100644 --- a/www/common/outer/userObject.js +++ b/www/common/outer/userObject.js @@ -694,7 +694,7 @@ define([ // Fix creation time if (!el.ctime) { el.ctime = el.atime; } // Fix title - if (!el.title) { el.title = Hash.getDefaultName(parsed); } + if (!el.title) { el.title = exp.getDefaultName(parsed); } // Fix channel if (!el.channel) { try { diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 3c13e0e8e..71282114b 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -50,9 +50,10 @@ define([ '/common/outer/local-store.js', '/customize/application_config.js', '/common/test.js', + '/common/userObject.js', ], waitFor(function (_CpNfOuter, _Cryptpad, _Crypto, _Cryptget, _SFrameChannel, _FilePicker, _Share, _Messaging, _Notifier, _Hash, _Util, _Realtime, - _Constants, _Feedback, _LocalStore, _AppConfig, _Test) { + _Constants, _Feedback, _LocalStore, _AppConfig, _Test, _UserObject) { CpNfOuter = _CpNfOuter; Cryptpad = _Cryptpad; Crypto = Utils.Crypto = _Crypto; @@ -68,6 +69,7 @@ define([ Utils.Constants = _Constants; Utils.Feedback = _Feedback; Utils.LocalStore = _LocalStore; + Utils.UserObject = _UserObject; AppConfig = _AppConfig; Test = _Test; @@ -271,7 +273,7 @@ define([ Utils.crypto = Utils.Crypto.createEncryptor(Utils.secret.keys); var parsed = Utils.Hash.parsePadUrl(window.location.href); if (!parsed.type) { throw new Error(); } - var defaultTitle = Utils.Hash.getDefaultName(parsed); + var defaultTitle = Utils.UserObject.getDefaultName(parsed); var edPublic, curvePublic, notifications, isTemplate; var forceCreationScreen = cfg.useCreationScreen && sessionStorage[Utils.Constants.displayPadCreationScreen]; @@ -1176,7 +1178,7 @@ define([ // Update metadata values and send new metadata inside parsed = Utils.Hash.parsePadUrl(window.location.href); - defaultTitle = Utils.Hash.getDefaultName(parsed); + defaultTitle = Utils.UserObject.getDefaultName(parsed); hashes = Utils.Hash.getHashes(secret); readOnly = false; updateMeta(); diff --git a/www/common/userObject.js b/www/common/userObject.js index 38372e880..48c74746b 100644 --- a/www/common/userObject.js +++ b/www/common/userObject.js @@ -15,8 +15,24 @@ define([ var TEMPLATE = module.TEMPLATE = "template"; var SHARED_FOLDERS = module.SHARED_FOLDERS = "sharedFolders"; + // Create untitled documents when no name is given + var getLocaleDate = function () { + if (window.Intl && window.Intl.DateTimeFormat) { + var options = {weekday: "short", year: "numeric", month: "long", day: "numeric"}; + return new window.Intl.DateTimeFormat(undefined, options).format(new Date()); + } + return new Date().toString().split(' ').slice(0,4).join(' '); + }; + module.getDefaultName = function (parsed) { + var type = parsed.type; + var name = (Messages.type)[type] + ' - ' + getLocaleDate(); + return name; + }; + module.init = function (files, config) { var exp = {}; + exp.getDefaultName = module.getDefaultName; + var sframeChan = config.sframeChan; var FILES_DATA = module.FILES_DATA = exp.FILES_DATA = Constants.storageKey; From 6049e677953d7b6c2eb2370d4678a486e096100b Mon Sep 17 00:00:00 2001 From: yflory Date: Thu, 5 Sep 2019 14:18:56 +0200 Subject: [PATCH 03/18] Fix remove yourself as owner with a new-ish account --- www/common/common-ui-elements.js | 1 + 1 file changed, 1 insertion(+) diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index bdfa56d7d..c0dfbe49b 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -124,6 +124,7 @@ define([ f = f || user; if (f.name) { f.displayName = f.name; + f.edPublic = edPublic; } } _owners[ed] = f || { From ed82936610b86deb9f0d2e6c3d982a6efd3c7107 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 5 Sep 2019 15:28:20 +0200 Subject: [PATCH 04/18] add nodejs support to common-hash --- www/common/common-hash.js | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/www/common/common-hash.js b/www/common/common-hash.js index 756877857..25bd981a8 100644 --- a/www/common/common-hash.js +++ b/www/common/common-hash.js @@ -1,10 +1,5 @@ -define([ - '/common/common-util.js', - '/bower_components/chainpad-crypto/crypto.js', - '/bower_components/tweetnacl/nacl-fast.min.js' -], function (Util, Crypto) { - var Nacl = window.nacl; - +(function () { +var factory = function (Util, Crypto, Nacl) { var Hash = window.CryptPad_Hash = {}; var uint8ArrayToHex = Util.uint8ArrayToHex; @@ -537,4 +532,19 @@ Version 1 }; return Hash; -}); +}; + + if (typeof(module) !== 'undefined' && module.exports) { + module.exports = factory(require("./common-util"), require("chainpad-crypto"), require("tweetnacl")); + } else if ((typeof(define) !== 'undefined' && define !== null) && (define.amd !== null)) { + define([ + '/common/common-util.js', + '/bower_components/chainpad-crypto/crypto.js', + '/bower_components/tweetnacl/nacl-fast.min.js' + ], function (Util, Crypto) { + factory(Util, Crypto, window.nacl); + }); + } else { + // unsupported initialization + } +}()); From 65b0112dcb42332738d5eb32d9e14cf21a770d9c Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 5 Sep 2019 15:35:01 +0200 Subject: [PATCH 05/18] batch concurrent requests for expensive IO and computations --- rpc.js | 194 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 106 insertions(+), 88 deletions(-) diff --git a/rpc.js b/rpc.js index 97b317754..c920f65c3 100644 --- a/rpc.js +++ b/rpc.js @@ -19,6 +19,7 @@ const getFolderSize = require("get-folder-size"); const Pins = require("./lib/pins"); const Meta = require("./lib/metadata"); const WriteQueue = require("./lib/write-queue"); +const BatchRead = require("./lib/batch-read"); var RPC = module.exports; @@ -231,6 +232,7 @@ var checkSignature = function (signedMsg, signature, publicKey) { return Nacl.sign.detached.verify(signedBuffer, signatureBuffer, pubBuffer); }; +const batchUserPins = BatchRead(); var loadUserPins = function (Env, publicKey, cb) { var session = getSession(Env.Sessions, publicKey); @@ -238,21 +240,23 @@ var loadUserPins = function (Env, publicKey, cb) { return cb(session.channels); } - var ref = {}; - var lineHandler = Pins.createLineHandler(ref, function (label, data) { - Log.error(label, { - log: publicKey, - data: data, + batchUserPins(publicKey, cb, function (done) { + var ref = {}; + var lineHandler = Pins.createLineHandler(ref, function (label, data) { + Log.error(label, { + log: publicKey, + data: data, + }); }); - }); - // if channels aren't in memory. load them from disk - Env.pinStore.getMessages(publicKey, lineHandler, function () { - // no more messages + // if channels aren't in memory. load them from disk + Env.pinStore.getMessages(publicKey, lineHandler, function () { + // no more messages - // only put this into the cache if it completes - session.channels = ref.pins; - cb(ref.pins); + // only put this into the cache if it completes + session.channels = ref.pins; + done(ref.pins); // FIXME no error handling? + }); }); }; @@ -290,45 +294,49 @@ var getUploadSize = function (Env, channel, cb) { }); }; +const batchFileSize = BatchRead(); var getFileSize = function (Env, channel, cb) { if (!isValidId(channel)) { return void cb('INVALID_CHAN'); } + batchFileSize(channel, cb, function (done) { + if (channel.length === 32) { + if (typeof(Env.msgStore.getChannelSize) !== 'function') { + return done('GET_CHANNEL_SIZE_UNSUPPORTED'); + } - if (channel.length === 32) { - if (typeof(Env.msgStore.getChannelSize) !== 'function') { - return cb('GET_CHANNEL_SIZE_UNSUPPORTED'); + return void Env.msgStore.getChannelSize(channel, function (e, size /*:number*/) { + if (e) { + if (e.code === 'ENOENT') { return void done(void 0, 0); } + return void done(e.code); + } + done(void 0, size); + }); } - return void Env.msgStore.getChannelSize(channel, function (e, size /*:number*/) { - if (e) { - if (e.code === 'ENOENT') { return void cb(void 0, 0); } - return void cb(e.code); - } - cb(void 0, size); + // 'channel' refers to a file, so you need another API + getUploadSize(Env, channel, function (e, size) { + if (typeof(size) === 'undefined') { return void done(e); } + done(void 0, size); }); - } - - // 'channel' refers to a file, so you need another API - getUploadSize(Env, channel, function (e, size) { - if (typeof(size) === 'undefined') { return void cb(e); } - cb(void 0, size); }); }; - +const batchMetadata = BatchRead(); var getMetadata = function (Env, channel, cb) { if (!isValidId(channel)) { return void cb('INVALID_CHAN'); } if (channel.length !== 32) { return cb("INVALID_CHAN"); } - var ref = {}; - var lineHandler = Meta.createLineHandler(ref, Log.error); + batchMetadata(channel, cb, function (done) { + var ref = {}; + var lineHandler = Meta.createLineHandler(ref, Log.error); - return void Env.msgStore.readChannelMetadata(channel, lineHandler, function (err) { - if (err) { - // stream errors? - return void cb(err); - } - cb(void 0, ref.meta); + return void Env.msgStore.readChannelMetadata(channel, lineHandler, function (err) { + if (err) { + // stream errors? + return void done(err); + } + done(void 0, ref.meta); + }); }); }; @@ -470,19 +478,22 @@ var getDeletedPads = function (Env, channels, cb) { }); }; +const batchTotalSize = BatchRead(); var getTotalSize = function (Env, publicKey, cb) { - var bytes = 0; - return void getChannelList(Env, publicKey, function (channels) { - if (!channels) { return cb('INVALID_PIN_LIST'); } // unexpected - - var count = channels.length; - if (!count) { cb(void 0, 0); } - - channels.forEach(function (channel) { - getFileSize(Env, channel, function (e, size) { - count--; - if (!e) { bytes += size; } - if (count === 0) { return cb(void 0, bytes); } + batchTotalSize(publicKey, cb, function (done) { + var bytes = 0; + return void getChannelList(Env, publicKey, function (channels) { + if (!channels) { return done('INVALID_PIN_LIST'); } // unexpected + + var count = channels.length; + if (!count) { return void done(void 0, 0); } + + channels.forEach(function (channel) { // FIXME this might as well be nThen + getFileSize(Env, channel, function (e, size) { + count--; + if (!e) { bytes += size; } + if (count === 0) { return done(void 0, bytes); } + }); }); }); }); @@ -1605,53 +1616,60 @@ var writePrivateMessage = function (Env, args, nfwssCtx, cb) { }); }; +const batchDiskUsage = BatchRead(); var getDiskUsage = function (Env, cb) { - var data = {}; - nThen(function (waitFor) { - getFolderSize('./', waitFor(function(err, info) { - data.total = info; - })); - getFolderSize(Env.paths.pin, waitFor(function(err, info) { - data.pin = info; - })); - getFolderSize(Env.paths.blob, waitFor(function(err, info) { - data.blob = info; - })); - getFolderSize(Env.paths.staging, waitFor(function(err, info) { - data.blobstage = info; - })); - getFolderSize(Env.paths.block, waitFor(function(err, info) { - data.block = info; - })); - getFolderSize(Env.paths.data, waitFor(function(err, info) { - data.datastore = info; - })); - }).nThen(function () { - cb (void 0, data); + batchDiskUsage('', cb, function (done) { + var data = {}; + nThen(function (waitFor) { + getFolderSize('./', waitFor(function(err, info) { + data.total = info; + })); + getFolderSize(Env.paths.pin, waitFor(function(err, info) { + data.pin = info; + })); + getFolderSize(Env.paths.blob, waitFor(function(err, info) { + data.blob = info; + })); + getFolderSize(Env.paths.staging, waitFor(function(err, info) { + data.blobstage = info; + })); + getFolderSize(Env.paths.block, waitFor(function(err, info) { + data.block = info; + })); + getFolderSize(Env.paths.data, waitFor(function(err, info) { + data.datastore = info; + })); + }).nThen(function () { + done(void 0, data); + }); }); }; + +const batchRegisteredUsers = BatchRead(); var getRegisteredUsers = function (Env, cb) { - var dir = Env.paths.pin; - var folders; - var users = 0; - nThen(function (waitFor) { - Fs.readdir(dir, waitFor(function (err, list) { - if (err) { - waitFor.abort(); - return void cb(err); - } - folders = list; - })); - }).nThen(function (waitFor) { - folders.forEach(function (f) { - var dir = Env.paths.pin + '/' + f; + batchRegisteredUsers('', cb, function (done) { + var dir = Env.paths.pin; + var folders; + var users = 0; + nThen(function (waitFor) { Fs.readdir(dir, waitFor(function (err, list) { - if (err) { return; } - users += list.length; + if (err) { + waitFor.abort(); + return void done(err); + } + folders = list; })); + }).nThen(function (waitFor) { + folders.forEach(function (f) { + var dir = Env.paths.pin + '/' + f; + Fs.readdir(dir, waitFor(function (err, list) { + if (err) { return; } + users += list.length; + })); + }); + }).nThen(function () { + done(void 0, users); }); - }).nThen(function () { - cb(void 0, users); }); }; var getActiveSessions = function (Env, ctx, cb) { From 023c29795a4fab876ff6ea8b04deaa02aefe1a28 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 5 Sep 2019 15:48:13 +0200 Subject: [PATCH 06/18] update grep-able comments in rpc --- rpc.js | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/rpc.js b/rpc.js index c920f65c3..9287d25ad 100644 --- a/rpc.js +++ b/rpc.js @@ -272,12 +272,12 @@ var getChannelList = function (Env, publicKey, cb) { }); }; -var makeFilePath = function (root, id) { +var makeFilePath = function (root, id) { // FIXME FILES if (typeof(id) !== 'string' || id.length <= 2) { return null; } return Path.join(root, id.slice(0, 2), id); }; -var getUploadSize = function (Env, channel, cb) { +var getUploadSize = function (Env, channel, cb) { // FIXME FILES var paths = Env.paths; var path = makeFilePath(paths.blob, channel); if (!path) { @@ -549,7 +549,7 @@ var applyCustomLimits = function (Env, config) { // The limits object contains storage limits for all the publicKey that have paid // To each key is associated an object containing the 'limit' value and a 'note' explaining that limit -var updateLimits = function (Env, config, publicKey, cb /*:(?string, ?any[])=>void*/) { +var updateLimits = function (Env, config, publicKey, cb /*:(?string, ?any[])=>void*/) { // FIXME BATCH? if (config.adminEmail === false) { applyCustomLimits(Env, config); @@ -842,7 +842,7 @@ var resetUserPins = function (Env, publicKey, channelList, cb) { }); }; -var makeFileStream = function (root, id, cb) { +var makeFileStream = function (root, id, cb) { // FIXME FILES var stub = id.slice(0, 2); var full = makeFilePath(root, id); if (!full) { @@ -873,7 +873,7 @@ var makeFileStream = function (root, id, cb) { }); }; -var isFile = function (filePath, cb) { +var isFile = function (filePath, cb) { // FIXME FILES /*:: if (typeof(filePath) !== 'string') { throw new Error('should never happen'); } */ Fs.stat(filePath, function (e, stats) { if (e) { @@ -903,8 +903,7 @@ var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { }); }; -var removeOwnedBlob = function (Env, blobId, unsafeKey, cb) { - // FIXME METADATA +var removeOwnedBlob = function (Env, blobId, unsafeKey, cb) { // FIXME FILES // FIXME METADATA var safeKey = escapeKeyCharacters(unsafeKey); var safeKeyPrefix = safeKey.slice(0,3); var blobPrefix = blobId.slice(0,2); @@ -1020,7 +1019,7 @@ var removePins = function (Env, safeKey, cb) { }); }; -var upload = function (Env, publicKey, content, cb) { +var upload = function (Env, publicKey, content, cb) { // FIXME FILES var paths = Env.paths; var dec; try { dec = Buffer.from(content, 'base64'); } @@ -1056,7 +1055,7 @@ var upload = function (Env, publicKey, content, cb) { } }; -var upload_cancel = function (Env, publicKey, fileSize, cb) { +var upload_cancel = function (Env, publicKey, fileSize, cb) { // FIXME FILES var paths = Env.paths; var session = getSession(Env.Sessions, publicKey); @@ -1080,7 +1079,7 @@ var upload_cancel = function (Env, publicKey, fileSize, cb) { }); }; -var upload_complete = function (Env, publicKey, id, cb) { // FIXME logging +var upload_complete = function (Env, publicKey, id, cb) { // FIXME FILES var paths = Env.paths; var session = getSession(Env.Sessions, publicKey); @@ -1096,7 +1095,7 @@ var upload_complete = function (Env, publicKey, id, cb) { // FIXME logging var oldPath = makeFilePath(paths.staging, publicKey); if (!oldPath) { - WARN('safeMkdir', "oldPath is null"); // FIXME logging + WARN('safeMkdir', "oldPath is null"); return void cb('RENAME_ERR'); } @@ -1104,13 +1103,13 @@ var upload_complete = function (Env, publicKey, id, cb) { // FIXME logging var prefix = id.slice(0, 2); var newPath = makeFilePath(paths.blob, id); if (typeof(newPath) !== 'string') { - WARN('safeMkdir', "newPath is null"); // FIXME logging + WARN('safeMkdir', "newPath is null"); return void cb('RENAME_ERR'); } Fse.mkdirp(Path.join(paths.blob, prefix), function (e) { if (e || !newPath) { - WARN('safeMkdir', e); // FIXME logging + WARN('safeMkdir', e); return void cb('RENAME_ERR'); } isFile(newPath, function (e, yes) { @@ -1133,7 +1132,6 @@ var upload_complete = function (Env, publicKey, id, cb) { // FIXME logging return void cb(e || 'PATH_ERR'); } - // lol wut handle ur errors Fse.move(oldPath, newPath, function (e) { if (e) { WARN('rename', e); @@ -1146,7 +1144,7 @@ var upload_complete = function (Env, publicKey, id, cb) { // FIXME logging tryLocation(handleMove); }; -/* +/* FIXME FILES var owned_upload_complete = function (Env, safeKey, cb) { var session = getSession(Env.Sessions, safeKey); @@ -1241,7 +1239,7 @@ var owned_upload_complete = function (Env, safeKey, cb) { }; */ -var owned_upload_complete = function (Env, safeKey, id, cb) { // FIXME logging +var owned_upload_complete = function (Env, safeKey, id, cb) { // FIXME FILES var session = getSession(Env.Sessions, safeKey); // the file has already been uploaded to the staging area @@ -1352,7 +1350,7 @@ var owned_upload_complete = function (Env, safeKey, id, cb) { // FIXME logging }); }; -var upload_status = function (Env, publicKey, filesize, cb) { +var upload_status = function (Env, publicKey, filesize, cb) { // FIXME FILES var paths = Env.paths; // validate that the provided size is actually a positive number @@ -1398,7 +1396,7 @@ var upload_status = function (Env, publicKey, filesize, cb) { author of the block, since we assume that the block will have been encrypted with xsalsa20-poly1305 which is authenticated. */ -var validateLoginBlock = function (Env, publicKey, signature, block, cb) { +var validateLoginBlock = function (Env, publicKey, signature, block, cb) { // FIXME BLOCKS // convert the public key to a Uint8Array and validate it if (typeof(publicKey) !== 'string') { return void cb('E_INVALID_KEY'); } @@ -1439,7 +1437,7 @@ var validateLoginBlock = function (Env, publicKey, signature, block, cb) { return void cb(null, u8_block); }; -var createLoginBlockPath = function (Env, publicKey) { +var createLoginBlockPath = function (Env, publicKey) { // FIXME BLOCKS // prepare publicKey to be used as a file name var safeKey = escapeKeyCharacters(publicKey); @@ -1453,7 +1451,7 @@ var createLoginBlockPath = function (Env, publicKey) { return Path.join(Env.paths.block, safeKey.slice(0, 2), safeKey); }; -var writeLoginBlock = function (Env, msg, cb) { +var writeLoginBlock = function (Env, msg, cb) { // FIXME BLOCKS //console.log(msg); var publicKey = msg[0]; var signature = msg[1]; @@ -1484,7 +1482,7 @@ var writeLoginBlock = function (Env, msg, cb) { cb(e); } })); - }).nThen(function () { // FIXME logging + }).nThen(function () { // actually write the block // flow is dumb and I need to guard against this which will never happen @@ -1508,7 +1506,7 @@ var writeLoginBlock = function (Env, msg, cb) { information, we can just sign some constant and use that as proof. */ -var removeLoginBlock = function (Env, msg, cb) { +var removeLoginBlock = function (Env, msg, cb) { // FIXME BLOCKS var publicKey = msg[0]; var signature = msg[1]; var block = Nacl.util.decodeUTF8('DELETE_BLOCK'); // clients and the server will have to agree on this constant @@ -1862,7 +1860,7 @@ RPC.create = function ( respond(e, [null, size, null]); }); case 'GET_METADATA': - return void getMetadata(Env, msg[1], function (e, data) { // FIXME METADATA + return void getMetadata(Env, msg[1], function (e, data) { WARN(e, msg[1]); respond(e, [null, data, null]); }); From d81ff791ad26b8957edd2e7e00ed1486b1160d54 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 5 Sep 2019 18:01:21 +0200 Subject: [PATCH 07/18] WIP cli tests for rpc --- lib/client/index.js | 13 ++++--- package.json | 2 +- scripts/test-rpc.js | 43 ----------------------- scripts/tests/index.js | 1 + scripts/tests/test-rpc.js | 74 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 85 insertions(+), 48 deletions(-) delete mode 100644 scripts/test-rpc.js create mode 100644 scripts/tests/index.js create mode 100644 scripts/tests/test-rpc.js diff --git a/lib/client/index.js b/lib/client/index.js index b2cfec437..fb5753058 100644 --- a/lib/client/index.js +++ b/lib/client/index.js @@ -24,13 +24,15 @@ var createNetwork = Client.createNetwork = function (url, cb) { var info = {}; Netflux.connect(url, function (url) { + // this websocket seems to never close properly if the error is + // ECONNREFUSED info.websocket = new WebSocket(url) .on('error', function (err) { - console.log(err); + CB(err); }) .on('close', function (err) { - console.log("close"); - console.log(err); + console.log("CLOSE_ERROR", err); + delete info.websocket; }); return info.websocket; }).then(function (network) { @@ -77,7 +79,10 @@ Client.create = function (config, cb) { if (config.network) { return; } // connect to the network... createNetwork('ws://localhost:3000/cryptpad_websocket', w(function (err, info) { - //console.log(_network); + if (err) { + w.abort(); + return void CB(err); + } config.network = info.network; config.websocket = info.websocket; })); diff --git a/package.json b/package.json index 8bd93b82d..562035f45 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "lint:less": "./node_modules/lesshint/bin/lesshint -c ./.lesshintrc ./customize.dist/src/less2/", "flow": "./node_modules/.bin/flow", "test": "node scripts/TestSelenium.js", - "test-rpc": "cd scripts && node test-rpc", + "test-rpc": "cd scripts/tests && node test-rpc", "template": "cd customize.dist/src && for page in ../index.html ../privacy.html ../terms.html ../about.html ../contact.html ../what-is-cryptpad.html ../features.html ../../www/login/index.html ../../www/register/index.html ../../www/user/index.html;do echo $page; cp template.html $page; done;" } } diff --git a/scripts/test-rpc.js b/scripts/test-rpc.js deleted file mode 100644 index 79177464d..000000000 --- a/scripts/test-rpc.js +++ /dev/null @@ -1,43 +0,0 @@ -/* globals process */ -var Client = require("../lib/client/"); -var Mailbox = require("../www/bower_components/chainpad-crypto").Mailbox; -var Nacl = require("tweetnacl"); - -var makeKeys = function () { - var pair = Nacl.box.keyPair(); - return { - curvePrivate: Nacl.util.encodeBase64(pair.secretKey), - curvePublic: Nacl.util.encodeBase64(pair.publicKey), - }; -}; - -Client.create(function (err, client) { - if (err) { - console.error(err); - process.exit(1); - } - - var channel = "d34ebe83931382fcad9fe2e2d0e2cb5f"; // channel - var recipient = "e8jvf36S3chzkkcaMrLSW7PPrz7VDp85lIFNI26dTmw="; // curvePublic - - var keys = makeKeys(); - var cryptor = Mailbox.createEncryptor(keys); - - var message = cryptor.encrypt(JSON.stringify({ - type: "CHEESE", - author: keys.curvePublic, - content: { - text: "CAMEMBERT", - } - }), recipient); - - client.anonRpc.send('WRITE_PRIVATE_MESSAGE', [channel, message], function (err, response) { - if (err) { - return void console.error(err); - } - - response = response; - // shutdown doesn't work, so we need to do this instead - client.shutdown(); - }); -}); diff --git a/scripts/tests/index.js b/scripts/tests/index.js new file mode 100644 index 000000000..934246d90 --- /dev/null +++ b/scripts/tests/index.js @@ -0,0 +1 @@ +require("./test-rpc"); diff --git a/scripts/tests/test-rpc.js b/scripts/tests/test-rpc.js new file mode 100644 index 000000000..2c355af3b --- /dev/null +++ b/scripts/tests/test-rpc.js @@ -0,0 +1,74 @@ +var Client = require("../../lib/client/"); +var Mailbox = require("../../www/bower_components/chainpad-crypto").Mailbox; +var Nacl = require("tweetnacl"); +var nThen = require("nthen"); + +var makeCurveKeys = function () { + var pair = Nacl.box.keyPair(); + return { + curvePrivate: Nacl.util.encodeBase64(pair.secretKey), + curvePublic: Nacl.util.encodeBase64(pair.publicKey), + }; +}; + +Client.create(function (err, client) { + if (err) { return void console.error(err); } + + nThen(function () { // BASIC KEY MANAGEMENT + // generate keys with login + // signing keys + // curve keys + // drive + }).nThen(function () { + // make a drive + // pin it + }).nThen(function () { // MAILBOXES + // write to your mailbox + // pin your mailbox + }).nThen(function () { + // create an owned pad + // pin the pad + // write to it + }).nThen(function () { + // get pinned usage + // remember the usage + }).nThen(function () { + // upload a file + // remember its size + }).nThen(function () { + // get pinned usage + // check that it is consistent with the size of your uploaded file + }).nThen(function () { + // delete your uploaded file + // unpin your owned file + }).nThen(function () { // EDITABLE METADATA + // + }).nThen(function () { + + }); + + var channel = "d34ebe83931382fcad9fe2e2d0e2cb5f"; // channel + var recipient = "e8jvf36S3chzkkcaMrLSW7PPrz7VDp85lIFNI26dTmw="; // curvePublic + + // curve keys + var keys = makeCurveKeys(); + var cryptor = Mailbox.createEncryptor(keys); + + var message = cryptor.encrypt(JSON.stringify({ + type: "CHEESE", + author: keys.curvePublic, + content: { + text: "CAMEMBERT", + } + }), recipient); + + client.anonRpc.send('WRITE_PRIVATE_MESSAGE', [channel, message], function (err, response) { + if (err) { + return void console.error(err); + } + + response = response; + // shutdown doesn't work, so we need to do this instead + client.shutdown(); + }); +}); From 53142c91e7ee471aa54cc63a1ec2f8c5922f4454 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 5 Sep 2019 18:02:00 +0200 Subject: [PATCH 08/18] fix broken requirejs support for common-hash --- www/common/common-hash.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/common/common-hash.js b/www/common/common-hash.js index 25bd981a8..199814577 100644 --- a/www/common/common-hash.js +++ b/www/common/common-hash.js @@ -542,7 +542,7 @@ Version 1 '/bower_components/chainpad-crypto/crypto.js', '/bower_components/tweetnacl/nacl-fast.min.js' ], function (Util, Crypto) { - factory(Util, Crypto, window.nacl); + return factory(Util, Crypto, window.nacl); }); } else { // unsupported initialization From 75b655e1e86c5d29fe98bb68268fb16f6e948e8a Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 6 Sep 2019 10:29:57 +0200 Subject: [PATCH 09/18] refactor rpc with support for multiple authenticated sessions --- www/common/common-util.js | 5 + www/common/rpc.js | 413 ++++++++++++++++++++++---------------- 2 files changed, 243 insertions(+), 175 deletions(-) diff --git a/www/common/common-util.js b/www/common/common-util.js index 22f20f710..580b9f539 100644 --- a/www/common/common-util.js +++ b/www/common/common-util.js @@ -1,5 +1,10 @@ (function (window) { var Util = {}; + + Util.tryParse = function (s) { + try { return JSON.parse(s); } catch (e) { return;} + }; + Util.mkAsync = function (f) { return function () { var args = Array.prototype.slice.call(arguments); diff --git a/www/common/rpc.js b/www/common/rpc.js index cefbfb666..e3c6a21a4 100644 --- a/www/common/rpc.js +++ b/www/common/rpc.js @@ -1,144 +1,198 @@ (function () { var factory = function (Util, Nacl) { + // we will send messages with a unique id for each RPC + // that id is returned with each response, indicating which call it was in response to var uid = Util.uid; + + // safely parse json messages, because they might cause parse errors + var tryParse = Util.tryParse; + + // we will sign various message with our edPrivate keys + // this handles that in a generic way var signMsg = function (data, signKey) { var buffer = Nacl.util.decodeUTF8(JSON.stringify(data)); return Nacl.util.encodeBase64(Nacl.sign.detached(buffer, signKey)); }; -/* -types of messages: - pin -> hash - unpin -> hash - getHash -> hash - getTotalSize -> bytes - getFileSize -> bytes -*/ - + // sendMsg takes a pre-formed message, does a little validation + // adds a transaction id to the message and stores its callback + // and finally sends it off to the historyKeeper, which delegates its + // processing to the RPC submodule var sendMsg = function (ctx, data, cb) { - var network = ctx.network; - var hkn = network.historyKeeper; - var txid = uid(); + // enforce async behaviour + setTimeout(function () { + if (typeof(cb) !== 'function') { + return console.error('expected callback'); + } - if (typeof(cb) !== 'function') { - return console.error('expected callback'); - } + var network = ctx.network; + var hkn = network.historyKeeper; + if (typeof(hkn) !== 'string') { return void cb("NO_HISTORY_KEEPER"); } - var pending = ctx.pending[txid] = function (err, response) { - cb(err, response); - }; - pending.data = data; - pending.called = 0; - return network.sendto(hkn, JSON.stringify([txid, data])); + var txid = uid(); + + var pending = ctx.pending[txid] = function (err, response) { + cb(err, response); + }; + pending.data = data; + pending.called = 0; + + return network.sendto(hkn, JSON.stringify([txid, data])); + }); }; - var parse = function (msg) { - try { - return JSON.parse(msg); - } catch (e) { - return null; - } + var matchesAnon = function (ctx, txid) { + if (!ctx.anon) { return false; } + if (typeof(ctx.anon.pending[txid]) !== 'function') { return false; } + return true; }; - var onMsg = function (ctx, msg) { - var parsed = parse(msg); + var handleAnon = function (ctx /* anon_ctx */, txid, body /* parsed messages without txid */) { + // if anon is handling it we know there's a pending callback + var pending = ctx.pending[txid]; + if (body[0] === 'ERROR') { pending(body[1]); } + else { pending(void 0, body.slice(1)); } + delete ctx.pending[txid]; + }; + + var onMsg = function (ctx /* network context */, msg /* string message */) { + if (typeof(msg) !== 'string') { + console.error("received non-string message [%s]", msg); + } + var parsed = tryParse(msg); if (!parsed) { return void console.error(new Error('could not parse message: %s', msg)); } // RPC messages are always arrays. if (!Array.isArray(parsed)) { return; } + // ignore FULL_HISTORY messages + if (/(FULL_HISTORY|HISTORY_RANGE)/.test(parsed[0])) { return; } + var txid = parsed[0]; + // txid must be a string, or this message is not meant for us if (typeof(txid) !== 'string') { return; } - var cookie = parsed[1]; - var pending = ctx.pending[txid]; - - if (!(parsed && parsed.slice)) { - // RPC responses are arrays. this message isn't meant for us. - return; + if (matchesAnon(ctx, txid)) { + return void handleAnon(ctx.anon, txid, parsed.slice(1)); } - if (/(FULL_HISTORY|HISTORY_RANGE)/.test(parsed[0])) { return; } - var response = parsed.slice(2); - - if (typeof(pending) === 'function') { - if (parsed[1] === 'ERROR') { - if (parsed[2] === 'NO_COOKIE') { - return void ctx.send('COOKIE', "", function (e) { - if (e) { - console.error(e); - return void pending(e); - } - - // resend the same command again - // give up if you've already tried resending - if (ctx.resend(txid)) { delete ctx.pending[txid]; } - }); + // iterate over authenticated rpc contexts and check if they are expecting + // a message with this txid + if (ctx.authenticated.some(function (rpc_ctx) { + var pending = rpc_ctx.pending[txid]; + // not meant for you + if (typeof(pending) !== 'function') { return false; } + + // if you're here, the message is for you... + + if (parsed[1] !== 'ERROR') { + // if the server sent you a new cookie, replace the old one + if (/\|/.test(parsed[1]) && rpc_ctx.cookie !== parsed[1]) { + rpc_ctx.cookie = parsed[1]; } + pending(void 0, parsed.slice(2)); - pending(parsed[2]); - delete ctx.pending[txid]; - return; - } else { - // update the cookie - if (/\|/.test(cookie)) { - if (ctx.cookie !== cookie) { - ctx.cookie = cookie; + // if successful, delete the callback... + delete rpc_ctx.pending[txid]; + // prevent further iteration + return true; + } + + // NO_COOKIE errors mean you failed to authenticate. + // request a new cookie and resend the query + if (parsed[2] === 'NO_COOKIE') { + return void ctx.send('COOKIE', "", function (e) { + if (e) { + console.error(e); + return void pending(e); } - } + + // resend the same command again + // give up if you've already tried resending + if (ctx.resend(txid)) { delete ctx.pending[txid]; } + }); } - pending(void 0, response); - // if successful, delete the callback... + // if you're here then your RPC passed authentication but had some other error + // call back with the error message + pending(parsed[2]); + // and delete the pending callback delete ctx.pending[txid]; + + // prevent further iteration + return true; + })) { + // the message was handled, so stop here return; } - // HACK to hide messages from the anon rpc - if (parsed.length !== 4 && parsed[1] !== 'ERROR') { - console.log(parsed); - console.error("received message [%s] for txid[%s] with no callback", msg, txid); - } + console.error("UNHANDLED RPC MESSAGE"); }; - var create = function (network, edPrivateKey, edPublicKey, cb) { - var signKey; + var networks = []; + var contexts = []; - try { - signKey = Nacl.util.decodeBase64(edPrivateKey); - if (signKey.length !== 64) { - throw new Error('private key did not match expected length of 64'); - } - } catch (err) { - return void cb(err); - } + var initNetworkContext = function (network) { + var ctx = { + network: network, + connected: true, + anon: undefined, + authenticated: [], + }; + networks.push(network); + contexts.push(ctx); - var pubBuffer; - try { - pubBuffer = Nacl.util.decodeBase64(edPublicKey); - if (pubBuffer.length !== 32) { - return void cb('expected public key to be 32 uint'); - } - } catch (err) { - return void cb(err); - } + // add listeners... + network.on('message', function (msg, sender) { + if (sender !== network.historyKeeper) { return; } + onMsg(ctx, msg); + }); + network.on('disconnect', function () { + ctx.connected = false; + if (ctx.anon) { ctx.anon.connected = false; } + ctx.authenticated.forEach(function (ctx) { + ctx.connected = false; + }); + }); + + network.on('reconnect', function () { + if (ctx.anon) { ctx.anon.connected = true; } + ctx.authenticated.forEach(function (ctx) { + ctx.connected = true; + }); + }); + return ctx; + }; + + var getNetworkContext = function (network) { + var i; + networks.some(function (current, j) { + if (network !== current) { return false; } + i = j; + return true; + }); + + if (contexts[i]) { return contexts[i]; } + return initNetworkContext(network); + }; + + var initAuthenticatedRpc = function (networkContext, keys) { var ctx = { - network: network, - timeouts: {}, // timeouts - pending: {}, // callbacks + network: networkContext.network, + publicKey: keys.publicKeyString, + timeouts: {}, + pending: {}, cookie: null, connected: true, }; var send = ctx.send = function (type, msg, cb) { if (!ctx.connected && type !== 'COOKIE') { - return void setTimeout(function () { - cb('DISCONNECTED'); - }); + return void Util.mkAsync(cb)("DISCONNECTED"); } // construct a signed message... @@ -150,9 +204,9 @@ types of messages: data.unshift(ctx.cookie); } - var sig = signMsg(data, signKey); + var sig = signMsg(data, keys.signKey); - data.unshift(edPublicKey); + data.unshift(keys.publicKeyString); data.unshift(sig); // [sig, edPublicKey, cookie, type, msg] @@ -169,7 +223,7 @@ types of messages: // update the cookie and signature... pending.data[2] = ctx.cookie; - pending.data[0] = signMsg(pending.data.slice(2), signKey); + pending.data[0] = signMsg(pending.data.slice(2), keys.signKey); try { return ctx.network.sendto(ctx.network.historyKeeper, JSON.stringify([txid, pending.data])); @@ -187,7 +241,7 @@ types of messages: } // construct an unsigned message - var data = [null, edPublicKey, null, type, msg]; + var data = [null, keys.publicKeyString, null, type, msg]; if (ctx.cookie && ctx.cookie.join) { data[2] = ctx.cookie.join('|'); } else { @@ -197,103 +251,101 @@ types of messages: return sendMsg(ctx, data, cb); }; - network.on('message', function (msg, sender) { - if (sender !== network.historyKeeper) { return; } - onMsg(ctx, msg); - }); + ctx.destroy = function () { + // clear all pending timeouts + Object.keys(ctx.timeouts).forEach(function (to) { + clearTimeout(to); + }); - network.on('disconnect', function () { - ctx.connected = false; - }); + // remove the ctx from the network's stack + var idx = networkContext.authenticated.indexOf(ctx); + if (idx === -1) { return; } + networkContext.authenticated.splice(idx, 1); + }; - network.on('reconnect', function () { - send('COOKIE', "", function (e) { - if (e) { return void cb(e); } - ctx.connected = true; - }); - }); + networkContext.authenticated.push(ctx); + return ctx; + }; - // network.onHistoryKeeperChange is defined in chainpad-netflux.js - // The function we pass will be called when the drive reconnects and - // chainpad-netflux detects a new history keeper id - if (network.onHistoryKeeperChange) { - network.onHistoryKeeperChange(function () { - send('COOKIE', "", function (e) { - if (e) { return void cb(e); } - ctx.connected = true; - }); - }); - } + var getAuthenticatedContext = function (networkContext, keys) { + if (!networkContext) { throw new Error('expected network context'); } - send('COOKIE', "", function (e) { - if (e) { return void cb(e); } - // callback to provide 'send' method to whatever needs it - cb(void 0, { send: send, }); + var publicKey = keys.publicKeyString; + + var i; + networkContext.authenticated.some(function (ctx, j) { + if (ctx.publicKey !== publicKey) { return false; } + i = j; + return true; }); + + if (networkContext.authenticated[i]) { return networkContext.authenticated[i]; } + + return initAuthenticatedRpc(networkContext, keys); }; - var onAnonMsg = function (ctx, msg) { - var parsed = parse(msg); + var create = function (network, edPrivateKey, edPublicKey, cb) { + if (typeof(cb) !== 'function') { throw new Error("expected callback"); } - if (!parsed) { - return void console.error(new Error('could not parse message: %s', msg)); + var signKey; + + try { + signKey = Nacl.util.decodeBase64(edPrivateKey); + if (signKey.length !== 64) { + throw new Error('private key did not match expected length of 64'); + } + } catch (err) { + return void cb(err); } - // RPC messages are always arrays. - if (!Array.isArray(parsed)) { return; } - var txid = parsed[0]; + try { + if (Nacl.util.decodeBase64(edPublicKey).length !== 32) { + return void cb('expected public key to be 32 uint'); + } + } catch (err) { return void cb(err); } - // txid must be a string, or this message is not meant for us - if (typeof(txid) !== 'string') { return; } + if (!network) { return void cb('NO_NETWORK'); } - var pending = ctx.pending[txid]; + // get or create a context for the provided network + var net_ctx = getNetworkContext(network); - if (!(parsed && parsed.slice)) { - // RPC responses are arrays. this message isn't meant for us. - return; - } - if (/FULL_HISTORY/.test(parsed[0])) { return; } - var response = parsed.slice(2); - - if (typeof(pending) === 'function') { - if (parsed[1] === 'ERROR') { - pending(parsed[2]); - delete ctx.pending[txid]; - return; - } - pending(void 0, response); + var rpc_ctx = getAuthenticatedContext(net_ctx, { + publicKeyString: edPublicKey, + signKey: signKey, + }); - // if successful, delete the callback... - delete ctx.pending[txid]; - return; - } - // HACK: filter out ugly messages we don't care about - if (typeof(msg) !== 'string') { - console.error("received message [%s] for txid[%s] with no callback", msg, txid); - } + rpc_ctx.send('COOKIE', "", function (e) { + if (e) { return void cb(e); } + // callback to provide 'send' method to whatever needs it + cb(void 0, { + send: rpc_ctx.send, + destroy: rpc_ctx.destroy, + }); + }); }; - var createAnonymous = function (network, cb) { + var initAnonRpc = function (networkContext) { var ctx = { - network: network, - timeouts: {}, // timeouts - pending: {}, // callbacks - cookie: null, + network: networkContext.network, + timeouts: {}, + pending: {}, connected: true, }; - var send = ctx.send = function (type, msg, cb) { + // any particular network will only ever need one anonymous rpc + networkContext.anon = ctx; + + ctx.send = function (type, msg, cb) { if (!ctx.connected) { return void setTimeout(function () { cb('DISCONNECTED'); }); } - // construct an unsigned message... var data = [type, msg]; - // [sig, edPublicKey, cookie, type, msg] + // [type, msg] return sendMsg(ctx, data, cb); }; @@ -314,21 +366,32 @@ types of messages: } }; - network.on('message', function (msg, sender) { - if (sender !== network.historyKeeper) { return; } - onAnonMsg(ctx, msg); - }); + ctx.destroy = function () { + // clear all pending timeouts + Object.keys(ctx.timeouts).forEach(function (to) { + clearTimeout(to); + }); - network.on('disconnect', function () { - ctx.connected = false; - }); + networkContext.anon = undefined; + }; - network.on('reconnect', function () { - ctx.connected = true; - }); + return ctx; + }; + + var getAnonContext = function (networkContext) { + return networkContext.anon || initAnonRpc(networkContext); + }; + + var createAnonymous = function (network, cb) { + if (typeof(cb) !== 'function') { throw new Error("expected callback"); } + if (!network) { return void cb('NO_NETWORK'); } + + // get or create a context for the provided network + var ctx = getAnonContext(getNetworkContext(network)); cb(void 0, { - send: send + send: ctx.send, + destroy: ctx.destroy, }); }; From dc078c1ca11155b0cd690b86eeccba1c18e72a2f Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 6 Sep 2019 13:35:36 +0200 Subject: [PATCH 10/18] test mailboxes and RPCs from the command line, fix some minor errors --- lib/client/index.js | 28 +---- package-lock.json | 20 +++- package.json | 3 +- scripts/tests/test-rpc.js | 225 ++++++++++++++++++++++++++++++++------ www/common/common-hash.js | 4 +- www/common/rpc.js | 62 +++++------ 6 files changed, 245 insertions(+), 97 deletions(-) diff --git a/lib/client/index.js b/lib/client/index.js index fb5753058..8faf8f4a2 100644 --- a/lib/client/index.js +++ b/lib/client/index.js @@ -3,19 +3,9 @@ var WebSocket = require("ws"); // jshint ignore:line var nThen = require("nthen"); var Util = require("../../www/common/common-util"); -var Rpc = require("../../www/common/rpc"); var Nacl = require("tweetnacl"); -var makeKeys = function () { - var keys = Nacl.sign.keyPair.fromSeed(Nacl.randomBytes(Nacl.sign.seedLength)); - return { - secret: Nacl.util.encodeBase64(keys.secretKey), - public: Nacl.util.encodeBase64(keys.publicKey), - }; -}; - - var Client = module.exports; var createNetwork = Client.createNetwork = function (url, cb) { @@ -30,8 +20,7 @@ var createNetwork = Client.createNetwork = function (url, cb) { .on('error', function (err) { CB(err); }) - .on('close', function (err) { - console.log("CLOSE_ERROR", err); + .on('close', function (/* err */) { delete info.websocket; }); return info.websocket; @@ -102,21 +91,6 @@ Client.create = function (config, cb) { w.abort(); CB(err); }); - }).nThen(function (w) { - // connect to the anonRpc - Rpc.createAnonymous(config.network, w(function (err, rpc) { - if (err) { - return void CB('ANON_RPC_CONNECT_ERR'); - } - client.anonRpc = rpc; - })); - var keys = makeKeys(); - Rpc.create(config.network, keys.secret, keys.public, w(function (err, rpc) { - if (err) { - return void CB('RPC_CONNECT_ERR'); - } - client.rpc = rpc; - })); }).nThen(function () { CB(void 0, client); }); diff --git a/package-lock.json b/package-lock.json index 4c32aff68..0f19280a1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -98,10 +98,24 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" }, + "chainpad-crypto": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/chainpad-crypto/-/chainpad-crypto-0.2.2.tgz", + "integrity": "sha512-7MJ7qPz/C4sJPsDhPMjdSRmliOCPoRO0XM1vUomcgXA6HINlW+if9AAt/H4q154nYhZ/b57njgC6cWgd/RDidg==", + "requires": { + "tweetnacl": "git://github.com/dchest/tweetnacl-js.git#v0.12.2" + }, + "dependencies": { + "tweetnacl": { + "version": "git://github.com/dchest/tweetnacl-js.git#8a21381d696acdc4e99c9f706f1ad23285795f79", + "from": "git://github.com/dchest/tweetnacl-js.git#v0.12.2" + } + } + }, "chainpad-server": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/chainpad-server/-/chainpad-server-3.0.3.tgz", - "integrity": "sha512-NRfV7FFBEYy4ZVX7h0P5znu55X8v5K4iGWeMGihkfWZLKu70GmCPUTwpBCP79dUvnCToKEa4/e8aoSPcvZC8pA==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/chainpad-server/-/chainpad-server-3.0.5.tgz", + "integrity": "sha512-USKOMSHsNjnme81Qy3nQ+ji9eCkBPokYH4T82LVHAI0aayTSCXcTPUDLVGDBCRqe8NsXU4io1WPXn1KiZwB8fA==", "requires": { "nthen": "^0.1.8", "pull-stream": "^3.6.9", diff --git a/package.json b/package.json index 562035f45..f14fda1aa 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "url": "git://github.com/xwiki-labs/cryptpad.git" }, "dependencies": { - "chainpad-server": "~3.0.2", + "chainpad-crypto": "^0.2.2", + "chainpad-server": "^3.0.5", "express": "~4.16.0", "fs-extra": "^7.0.0", "get-folder-size": "^2.0.1", diff --git a/scripts/tests/test-rpc.js b/scripts/tests/test-rpc.js index 2c355af3b..54393efbb 100644 --- a/scripts/tests/test-rpc.js +++ b/scripts/tests/test-rpc.js @@ -1,7 +1,38 @@ +/* globals process */ + var Client = require("../../lib/client/"); var Mailbox = require("../../www/bower_components/chainpad-crypto").Mailbox; var Nacl = require("tweetnacl"); var nThen = require("nthen"); +var Rpc = require("../../www/common/rpc"); +var Hash = require("../../www/common/common-hash"); +var CpNetflux = require("../../www/bower_components/chainpad-netflux"); + +var createMailbox = function (config, cb) { + var webchannel; + + CpNetflux.start({ + network: config.network, + channel: config.channel, + crypto: config.crypto, + owners: [ config.edPublic ], + + noChainPad: true, + onConnect: function (wc /*, sendMessage */) { + webchannel = wc; + }, + onMessage: function (/* msg, user, vKey, isCp, hash, author */) { + + }, + onReady: function () { + cb(void 0, webchannel); + }, + }); +}; + +process.on('unhandledRejection', function (err) { + console.error(err); +}); var makeCurveKeys = function () { var pair = Nacl.box.keyPair(); @@ -11,14 +42,166 @@ var makeCurveKeys = function () { }; }; -Client.create(function (err, client) { - if (err) { return void console.error(err); } +var makeEdKeys = function () { + var keys = Nacl.sign.keyPair.fromSeed(Nacl.randomBytes(Nacl.sign.seedLength)); + return { + edPrivate: Nacl.util.encodeBase64(keys.secretKey), + edPublic: Nacl.util.encodeBase64(keys.publicKey), + }; +}; + +var EMPTY_ARRAY_HASH = 'slspTLTetp6gCkw88xE5BIAbYBXllWvQGahXCx/h1gQOlE7zze4W0KRlA8puZZol8hz5zt3BPzUqPJgTjBXWrw=='; + +var createUser = function (config, cb) { + // config should contain keys for a team rpc (ed) + // teamEdKeys + + var user; + nThen(function (w) { + Client.create(w(function (err, client) { + if (err) { + w.abort(); + return void cb(err); + } + user = client; + })); + }).nThen(function (w) { + // make all the parameters you'll need + + var network = user.network = user.config.network; + user.edKeys = makeEdKeys(); + + user.curveKeys = makeCurveKeys(); + user.mailbox = Mailbox.createEncryptor(user.curveKeys); + user.mailboxChannel = Hash.createChannelId(); + + // create an anon rpc for alice + Rpc.createAnonymous(network, w(function (err, rpc) { + if (err) { + w.abort(); + user.shutdown(); + return void console.error('ANON_RPC_CONNECT_ERR'); + } + user.anonRpc = rpc; + })); + + Rpc.create(network, user.edKeys.edPrivate, user.edKeys.edPublic, w(function (err, rpc) { + if (err) { + w.abort(); + user.shutdown(); + console.error(err); + return console.log('RPC_CONNECT_ERR'); + } + user.rpc = rpc; + })); + + Rpc.create(network, config.teamEdKeys.edPrivate, config.teamEdKeys.edPublic, w(function (err, rpc) { + if (err) { + w.abort(); + user.shutdown(); + return console.log('RPC_CONNECT_ERR'); + } + user.team_rpc = rpc; + })); + }).nThen(function (w) { + // some basic sanity checks... + user.rpc.send('GET_HASH', user.edKeys.edPublic, w(function (err, hash) { + if (err) { + w.abort(); + return void cb(err); + } + + if (!hash || hash[0] !== EMPTY_ARRAY_HASH) { + console.error("EXPECTED EMPTY ARRAY HASH"); + process.exit(1); + } + })); + }).nThen(function (w) { + // create and subscribe to your mailbox + createMailbox({ + network: user.network, + channel: user.mailboxChannel, + crypto: user.mailbox, + edPublic: user.edKeys.edPublic, + }, w(function (err, wc) { + if (err) { + w.abort(); + console.error("Mailbox creation error"); + process.exit(1); + } + wc.leave(); + })); + }).nThen(function (w) { + // confirm that you own your mailbox + user.anonRpc.send("GET_METADATA", user.mailboxChannel, w(function (err, data) { + if (err) { + w.abort(); + return void cb(err); + } + try { + if (data[0].owners[0] !== user.edKeys.edPublic) { + throw new Error("INCORRECT MAILBOX OWNERSHIP METADATA"); + } + } catch (err2) { + w.abort(); + return void cb(err2); + } + })); + }).nThen(function () { + + }).nThen(function () { + + cb(void 0, user); + }); +}; + +var alice, bob; + +nThen(function (w) { + var sharedConfig = { + teamEdKeys: makeEdKeys(), + }; + + createUser(sharedConfig, w(function (err, _alice) { + if (err) { + w.abort(); + return void console.log(err); + } + alice = _alice; + })); + createUser(sharedConfig, w(function (err, _bob) { + if (err) { + w.abort(); + return void console.log(err); + } + bob = _bob; + })); +}).nThen(function (w) { + // Alice sends a message to Bob's mailbox + + + var message = alice.mailbox.encrypt(JSON.stringify({ + type: "CHEESE", + author: alice.curveKeys.curvePublic, + content: { + text: "CAMEMBERT", + } + }), bob.curveKeys.curvePublic); + + alice.anonRpc.send('WRITE_PRIVATE_MESSAGE', [bob.mailboxChannel, message], w(function (err, response) { + if (err) { + return void console.error(err); + } + + // XXX validate that the write was actually successful by checking its size + + response = response; + // shutdown doesn't work, so we need to do this instead + })); +}).nThen(function () { + + nThen(function () { - nThen(function () { // BASIC KEY MANAGEMENT - // generate keys with login - // signing keys - // curve keys - // drive }).nThen(function () { // make a drive // pin it @@ -46,29 +229,9 @@ Client.create(function (err, client) { }).nThen(function () { }); +}).nThen(function () { + alice.shutdown(); + bob.shutdown(); +}); - var channel = "d34ebe83931382fcad9fe2e2d0e2cb5f"; // channel - var recipient = "e8jvf36S3chzkkcaMrLSW7PPrz7VDp85lIFNI26dTmw="; // curvePublic - - // curve keys - var keys = makeCurveKeys(); - var cryptor = Mailbox.createEncryptor(keys); - - var message = cryptor.encrypt(JSON.stringify({ - type: "CHEESE", - author: keys.curvePublic, - content: { - text: "CAMEMBERT", - } - }), recipient); - - client.anonRpc.send('WRITE_PRIVATE_MESSAGE', [channel, message], function (err, response) { - if (err) { - return void console.error(err); - } - response = response; - // shutdown doesn't work, so we need to do this instead - client.shutdown(); - }); -}); diff --git a/www/common/common-hash.js b/www/common/common-hash.js index 199814577..5c1c1a81d 100644 --- a/www/common/common-hash.js +++ b/www/common/common-hash.js @@ -1,4 +1,4 @@ -(function () { +(function (window) { var factory = function (Util, Crypto, Nacl) { var Hash = window.CryptPad_Hash = {}; @@ -547,4 +547,4 @@ Version 1 } else { // unsupported initialization } -}()); +}(typeof(window) !== 'undefined'? window : {})); diff --git a/www/common/rpc.js b/www/common/rpc.js index e3c6a21a4..75ea7a0de 100644 --- a/www/common/rpc.js +++ b/www/common/rpc.js @@ -19,26 +19,21 @@ var factory = function (Util, Nacl) { // and finally sends it off to the historyKeeper, which delegates its // processing to the RPC submodule var sendMsg = function (ctx, data, cb) { - // enforce async behaviour - setTimeout(function () { - if (typeof(cb) !== 'function') { - return console.error('expected callback'); - } + if (typeof(cb) !== 'function') { throw new Error('expected callback'); } - var network = ctx.network; - var hkn = network.historyKeeper; - if (typeof(hkn) !== 'string') { return void cb("NO_HISTORY_KEEPER"); } + var network = ctx.network; + var hkn = network.historyKeeper; + if (typeof(hkn) !== 'string') { return void cb("NO_HISTORY_KEEPER"); } - var txid = uid(); + var txid = uid(); - var pending = ctx.pending[txid] = function (err, response) { - cb(err, response); - }; - pending.data = data; - pending.called = 0; + var pending = ctx.pending[txid] = function (err, response) { + cb(err, response); + }; + pending.data = data; + pending.called = 0; - return network.sendto(hkn, JSON.stringify([txid, data])); - }); + return network.sendto(hkn, JSON.stringify([txid, data])); }; var matchesAnon = function (ctx, txid) { @@ -190,9 +185,11 @@ var factory = function (Util, Nacl) { connected: true, }; - var send = ctx.send = function (type, msg, cb) { + var send = ctx.send = function (type, msg, _cb) { + var cb = Util.mkAsync(_cb); + if (!ctx.connected && type !== 'COOKIE') { - return void Util.mkAsync(cb)("DISCONNECTED"); + return void cb("DISCONNECTED"); } // construct a signed message... @@ -233,12 +230,9 @@ var factory = function (Util, Nacl) { } }; - send.unauthenticated = function (type, msg, cb) { - if (!ctx.connected) { - return void setTimeout(function () { - cb('DISCONNECTED'); - }); - } + send.unauthenticated = function (type, msg, _cb) { + var cb = Util.mkAsync(_cb); + if (!ctx.connected) { return void cb('DISCONNECTED'); } // construct an unsigned message var data = [null, keys.publicKeyString, null, type, msg]; @@ -284,8 +278,10 @@ var factory = function (Util, Nacl) { return initAuthenticatedRpc(networkContext, keys); }; - var create = function (network, edPrivateKey, edPublicKey, cb) { - if (typeof(cb) !== 'function') { throw new Error("expected callback"); } + var create = function (network, edPrivateKey, edPublicKey, _cb) { + if (typeof(_cb) !== 'function') { throw new Error("expected callback"); } + + var cb = Util.mkAsync(_cb); var signKey; @@ -335,12 +331,9 @@ var factory = function (Util, Nacl) { // any particular network will only ever need one anonymous rpc networkContext.anon = ctx; - ctx.send = function (type, msg, cb) { - if (!ctx.connected) { - return void setTimeout(function () { - cb('DISCONNECTED'); - }); - } + ctx.send = function (type, msg, _cb) { + var cb = Util.mkAsync(_cb); + if (!ctx.connected) { return void cb('DISCONNECTED'); } // construct an unsigned message... var data = [type, msg]; @@ -382,7 +375,10 @@ var factory = function (Util, Nacl) { return networkContext.anon || initAnonRpc(networkContext); }; - var createAnonymous = function (network, cb) { + var createAnonymous = function (network, _cb) { + // enforce asynchrony + var cb = Util.mkAsync(_cb); + if (typeof(cb) !== 'function') { throw new Error("expected callback"); } if (!network) { return void cb('NO_NETWORK'); } From fe884278f429120802a3a0043dc77904af0783ad Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 6 Sep 2019 13:36:06 +0200 Subject: [PATCH 11/18] ignore non-interesting errors when processing metadata --- lib/metadata.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/metadata.js b/lib/metadata.js index 63ab31819..73f82b6ea 100644 --- a/lib/metadata.js +++ b/lib/metadata.js @@ -195,6 +195,10 @@ Meta.createLineHandler = function (ref, errorHandler) { return function (err, line) { if (err) { + // it's not abnormal that metadata exists without a corresponding log + // so ENOENT is fine + if (ref.index === 0 && err.code === 'ENOENT') { return; } + // any other errors are abnormal return void errorHandler('METADATA_HANDLER_LINE_ERR', { error: err, index: ref.index, From ee3ac1cd56734f237b03743c4fac2cb83bd52479 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 6 Sep 2019 13:36:56 +0200 Subject: [PATCH 12/18] drop unnecessary lint compliance --- www/common/outer/mailbox.js | 1 - 1 file changed, 1 deletion(-) diff --git a/www/common/outer/mailbox.js b/www/common/outer/mailbox.js index ac8c1aa95..30c03c1b9 100644 --- a/www/common/outer/mailbox.js +++ b/www/common/outer/mailbox.js @@ -189,7 +189,6 @@ proxy.mailboxes = { box.queue.push(msg); } }; - Crypto = Crypto; if (!Crypto.Mailbox) { return void console.error("chainpad-crypto is outdated and doesn't support mailboxes."); } From d375ee8107402d8cc3e5b6e13d5271c30bc58593 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 6 Sep 2019 14:08:25 +0200 Subject: [PATCH 13/18] fix some reference errors that weren't triggered unless disconnected --- www/common/rpc.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/www/common/rpc.js b/www/common/rpc.js index 75ea7a0de..044ee0aa2 100644 --- a/www/common/rpc.js +++ b/www/common/rpc.js @@ -99,7 +99,7 @@ var factory = function (Util, Nacl) { // NO_COOKIE errors mean you failed to authenticate. // request a new cookie and resend the query if (parsed[2] === 'NO_COOKIE') { - return void ctx.send('COOKIE', "", function (e) { + return void rpc_ctx.send('COOKIE', "", function (e) { if (e) { console.error(e); return void pending(e); @@ -107,7 +107,7 @@ var factory = function (Util, Nacl) { // resend the same command again // give up if you've already tried resending - if (ctx.resend(txid)) { delete ctx.pending[txid]; } + if (rpc_ctx.resend(txid)) { delete rpc_ctx.pending[txid]; } }); } @@ -115,7 +115,7 @@ var factory = function (Util, Nacl) { // call back with the error message pending(parsed[2]); // and delete the pending callback - delete ctx.pending[txid]; + delete rpc_ctx.pending[txid]; // prevent further iteration return true; From a7e40040a130b4424634ac9a38135dfaad8dd182 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 6 Sep 2019 14:36:22 +0200 Subject: [PATCH 14/18] log unauthenticated RPC calls as silly --- rpc.js | 1 + 1 file changed, 1 insertion(+) diff --git a/rpc.js b/rpc.js index 9287d25ad..5655061a7 100644 --- a/rpc.js +++ b/rpc.js @@ -1837,6 +1837,7 @@ RPC.create = function ( }; var handleUnauthenticatedMessage = function (msg, respond, nfwssCtx) { + Log.silly('LOG_RPC', msg[0]); switch (msg[0]) { case 'GET_HISTORY_OFFSET': { if (typeof(msg[1]) !== 'object' || typeof(msg[1].channelName) !== 'string') { From 340a290ceb632e52b321266fb1e18f947e75ffc0 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 6 Sep 2019 14:58:00 +0200 Subject: [PATCH 15/18] add names for batched reads for debugging --- historyKeeper.js | 2 +- rpc.js | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/historyKeeper.js b/historyKeeper.js index b84bb8bc3..d90b8c113 100644 --- a/historyKeeper.js +++ b/historyKeeper.js @@ -232,7 +232,7 @@ module.exports.create = function (cfg) { as an added bonus: if the channel exists but its index does not then it caches the index */ - const batchIndexReads = BatchRead(); + const batchIndexReads = BatchRead("HK_GET_INDEX"); const getIndex = (ctx, channelName, cb) => { const chan = ctx.channels[channelName]; // if there is a channel in memory and it has an index cached, return it diff --git a/rpc.js b/rpc.js index 5655061a7..71e267dfd 100644 --- a/rpc.js +++ b/rpc.js @@ -232,7 +232,7 @@ var checkSignature = function (signedMsg, signature, publicKey) { return Nacl.sign.detached.verify(signedBuffer, signatureBuffer, pubBuffer); }; -const batchUserPins = BatchRead(); +const batchUserPins = BatchRead("LOAD_USER_PINS"); var loadUserPins = function (Env, publicKey, cb) { var session = getSession(Env.Sessions, publicKey); @@ -294,7 +294,7 @@ var getUploadSize = function (Env, channel, cb) { // FIXME FILES }); }; -const batchFileSize = BatchRead(); +const batchFileSize = BatchRead("GET_FILE_SIZE"); var getFileSize = function (Env, channel, cb) { if (!isValidId(channel)) { return void cb('INVALID_CHAN'); } batchFileSize(channel, cb, function (done) { @@ -320,7 +320,7 @@ var getFileSize = function (Env, channel, cb) { }); }; -const batchMetadata = BatchRead(); +const batchMetadata = BatchRead("GET_METADATA"); var getMetadata = function (Env, channel, cb) { if (!isValidId(channel)) { return void cb('INVALID_CHAN'); } @@ -478,7 +478,7 @@ var getDeletedPads = function (Env, channels, cb) { }); }; -const batchTotalSize = BatchRead(); +const batchTotalSize = BatchRead("GET_TOTAL_SIZE"); var getTotalSize = function (Env, publicKey, cb) { batchTotalSize(publicKey, cb, function (done) { var bytes = 0; @@ -1614,7 +1614,7 @@ var writePrivateMessage = function (Env, args, nfwssCtx, cb) { }); }; -const batchDiskUsage = BatchRead(); +const batchDiskUsage = BatchRead("GET_DISK_USAGE"); var getDiskUsage = function (Env, cb) { batchDiskUsage('', cb, function (done) { var data = {}; @@ -1643,7 +1643,7 @@ var getDiskUsage = function (Env, cb) { }); }; -const batchRegisteredUsers = BatchRead(); +const batchRegisteredUsers = BatchRead("GET_REGISTERED_USERS"); var getRegisteredUsers = function (Env, cb) { batchRegisteredUsers('', cb, function (done) { var dir = Env.paths.pin; From cb3df0552b9affdf11026eaffd60b59179cd1c04 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 6 Sep 2019 14:59:32 +0200 Subject: [PATCH 16/18] generate new txids when resending RPC commands and prevent unintended fallthrough in handler --- www/common/rpc.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/www/common/rpc.js b/www/common/rpc.js index 044ee0aa2..cb7790602 100644 --- a/www/common/rpc.js +++ b/www/common/rpc.js @@ -99,7 +99,7 @@ var factory = function (Util, Nacl) { // NO_COOKIE errors mean you failed to authenticate. // request a new cookie and resend the query if (parsed[2] === 'NO_COOKIE') { - return void rpc_ctx.send('COOKIE', "", function (e) { + rpc_ctx.send('COOKIE', "", function (e) { if (e) { console.error(e); return void pending(e); @@ -109,6 +109,8 @@ var factory = function (Util, Nacl) { // give up if you've already tried resending if (rpc_ctx.resend(txid)) { delete rpc_ctx.pending[txid]; } }); + // prevent further iteration + return true; } // if you're here then your RPC passed authentication but had some other error @@ -124,7 +126,7 @@ var factory = function (Util, Nacl) { return; } - console.error("UNHANDLED RPC MESSAGE"); + console.error("UNHANDLED RPC MESSAGE", msg); }; var networks = []; @@ -221,9 +223,16 @@ var factory = function (Util, Nacl) { // update the cookie and signature... pending.data[2] = ctx.cookie; pending.data[0] = signMsg(pending.data.slice(2), keys.signKey); + + // store the callback with a new txid + var new_txid = uid(); + ctx.pending[new_txid] = pending; + // and delete the old one + delete ctx.pending[txid]; + try { return ctx.network.sendto(ctx.network.historyKeeper, - JSON.stringify([txid, pending.data])); + JSON.stringify([new_txid, pending.data])); } catch (e) { console.log("failed to resend"); console.error(e); From 29813e485d8dc234d3a1e7ae599f090cad9de9d2 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 6 Sep 2019 15:03:42 +0200 Subject: [PATCH 17/18] add a little debugging info to batch-read --- lib/batch-read.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/batch-read.js b/lib/batch-read.js index 66f106c54..3e729e66d 100644 --- a/lib/batch-read.js +++ b/lib/batch-read.js @@ -39,7 +39,7 @@ read('./pewpew.txt', function (err, data) { */ -module.exports = function () { +module.exports = function (/* task */) { var map = {}; return function (id, cb, impl) { if (typeof(cb) !== 'function' || typeof(impl) !== 'function') { @@ -49,6 +49,9 @@ module.exports = function () { map[id] = [cb]; impl(function () { var args = Array.prototype.slice.call(arguments); + + //if (map[id] && map[id].length > 1) { console.log("BATCH-READ DID ITS JOB for [%s][%s]", task, id); } + map[id].forEach(function (h) { h.apply(null, args); }); From ecfac9e7b34e165113d7d54ee0ed4c5f6c9b326a Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 6 Sep 2019 15:18:55 +0200 Subject: [PATCH 18/18] test usage of multiple rpc sessions with the same network --- scripts/tests/test-rpc.js | 64 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/scripts/tests/test-rpc.js b/scripts/tests/test-rpc.js index 54393efbb..8c9f3348c 100644 --- a/scripts/tests/test-rpc.js +++ b/scripts/tests/test-rpc.js @@ -147,9 +147,70 @@ var createUser = function (config, cb) { return void cb(err2); } })); + }).nThen(function (w) { + // pin your mailbox + user.rpc.send('PIN', [user.mailboxChannel], w(function (err, data) { + if (err) { + w.abort(); + return void cb(err); + } + try { + if (data[0] === EMPTY_ARRAY_HASH) { throw new Error("PIN_DIDNT_WORK"); } + user.latestPinHash = data[0]; + } catch (err2) { + w.abort(); + return void cb(err2); + } + })); + }).nThen(function (w) { + user.team_rpc.send('GET_HASH', config.teamEdKeys.edPublic, w(function (err, hash) { + if (err) { + w.abort(); + return void cb(err); + } + if (!hash || hash[0] !== EMPTY_ARRAY_HASH) { + console.error("EXPECTED EMPTY ARRAY HASH"); + process.exit(1); + } + })); }).nThen(function () { + // TODO check your quota usage + }).nThen(function (w) { + user.rpc.send('UNPIN', [user.mailboxChannel], w(function (err, data) { + if (err) { + w.abort(); + return void cb(err); + } + try { + if (data[0] !== EMPTY_ARRAY_HASH) { throw new Error("UNPIN_DIDNT_WORK"); } + user.latestPinHash = data[0]; + } catch (err2) { + w.abort(); + return void cb(err2); + } + })); + }).nThen(function (w) { + // clean up the pin list to avoid lots of accounts on the server + user.rpc.send("REMOVE_PINS", undefined, w(function (err, data) { + if (err) { + w.abort(); + return void cb(err); + } + if (!data || data[0] !== 'OK') { + w.abort(); + return void cb("REMOVE_PINS_DIDNT_WORK"); + } + })); }).nThen(function () { + user.cleanup = function (cb) { + // TODO remove your mailbox + + cb = cb; + }; + + + cb(void 0, user); }); @@ -177,9 +238,6 @@ nThen(function (w) { bob = _bob; })); }).nThen(function (w) { - // Alice sends a message to Bob's mailbox - - var message = alice.mailbox.encrypt(JSON.stringify({ type: "CHEESE", author: alice.curveKeys.curvePublic,