From 2df65ed446e5cc3dbd7f65d2d0ef1966ae92aef1 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 13 Feb 2020 17:51:02 -0500 Subject: [PATCH 01/14] implement 'mkTimeout' method in common-util --- www/common/common-util.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/www/common/common-util.js b/www/common/common-util.js index 1af32103f..da373f1ad 100644 --- a/www/common/common-util.js +++ b/www/common/common-util.js @@ -68,6 +68,19 @@ }; }; + Util.mkTimeout = function (_f, ms) { + ms = ms || 0; + var f = Util.once(_f); + + var timeout = setTimeout(function () { + f('TIMEOUT'); + }, ms); + + return Util.both(f, function () { + clearTimeout(timeout); + }); + }; + Util.response = function () { var pending = {}; var timeouts = {}; From 1fc8c1de164683b543f4c9fed8cc5ae29f00859e Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 08:48:15 -0500 Subject: [PATCH 02/14] add missing connect-src directives to example nginx conf --- docs/example.nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/example.nginx.conf b/docs/example.nginx.conf index 6d45b8198..80c799f8c 100644 --- a/docs/example.nginx.conf +++ b/docs/example.nginx.conf @@ -72,7 +72,7 @@ server { set $styleSrc "'unsafe-inline' 'self' ${main_domain}"; # connect-src restricts URLs which can be loaded using script interfaces - set $connectSrc "'self' https://${main_domain} $main_domain https://${api_domain} blob:"; + set $connectSrc "'self' https://${main_domain} ${main_domain} https://${api_domain} blob: wss://${api_domain} ${api_domain} ${files_domain}"; # fonts can be loaded from data-URLs or the main domain set $fontSrc "'self' data: ${main_domain}"; From 8700345ccc0c9d69e96bff7cd5304a44b0d62011 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 08:48:15 -0500 Subject: [PATCH 03/14] add missing connect-src directives to example nginx conf --- docs/example.nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/example.nginx.conf b/docs/example.nginx.conf index 6d45b8198..80c799f8c 100644 --- a/docs/example.nginx.conf +++ b/docs/example.nginx.conf @@ -72,7 +72,7 @@ server { set $styleSrc "'unsafe-inline' 'self' ${main_domain}"; # connect-src restricts URLs which can be loaded using script interfaces - set $connectSrc "'self' https://${main_domain} $main_domain https://${api_domain} blob:"; + set $connectSrc "'self' https://${main_domain} ${main_domain} https://${api_domain} blob: wss://${api_domain} ${api_domain} ${files_domain}"; # fonts can be loaded from data-URLs or the main domain set $fontSrc "'self' data: ${main_domain}"; From f86196e40adfccade922e63bba5dafbff20538fe Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 09:01:10 -0500 Subject: [PATCH 04/14] implement shared environment between historyKeeper and RPC --- lib/commands/channel.js | 10 ++-- lib/commands/metadata.js | 4 +- lib/historyKeeper.js | 97 ++++++++++++++++++++++++++++-- lib/rpc.js | 125 +++++---------------------------------- 4 files changed, 114 insertions(+), 122 deletions(-) diff --git a/lib/commands/channel.js b/lib/commands/channel.js index 88404a9b2..6296aee0e 100644 --- a/lib/commands/channel.js +++ b/lib/commands/channel.js @@ -23,7 +23,7 @@ Channel.clearOwnedChannel = function (Env, safeKey, channelId, cb, Server) { if (e) { return void cb(e); } cb(); - const channel_cache = Env.historyKeeper.channel_cache; + const channel_cache = Env.channel_cache; const clear = function () { // delete the channel cache because it will have been invalidated @@ -117,8 +117,8 @@ Channel.removeOwnedChannel = function (Env, safeKey, channelId, cb, Server) { } cb(void 0, 'OK'); - const channel_cache = Env.historyKeeper.channel_cache; - const metadata_cache = Env.historyKeeper.metadata_cache; + const channel_cache = Env.channel_cache; + const metadata_cache = Env.metadata_cache; const clear = function () { delete channel_cache[channelId]; @@ -187,8 +187,8 @@ Channel.trimHistory = function (Env, safeKey, data, cb) { // clear historyKeeper's cache for this channel Env.historyKeeper.channelClose(channelId); cb(void 0, 'OK'); - delete Env.historyKeeper.channel_cache[channelId]; - delete Env.historyKeeper.metadata_cache[channelId]; + delete Env.channel_cache[channelId]; + delete Env.metadata_cache[channelId]; }); }); }; diff --git a/lib/commands/metadata.js b/lib/commands/metadata.js index 41aea9888..511f963bf 100644 --- a/lib/commands/metadata.js +++ b/lib/commands/metadata.js @@ -111,8 +111,8 @@ Data.setMetadata = function (Env, safeKey, data, cb, Server) { cb(void 0, metadata); next(); - const metadata_cache = Env.historyKeeper.metadata_cache; - const channel_cache = Env.historyKeeper.channel_cache; + const metadata_cache = Env.metadata_cache; + const channel_cache = Env.channel_cache; metadata_cache[channel] = metadata; diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 03f31383b..e99b279eb 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -7,8 +7,20 @@ const BatchRead = require("./batch-read"); const RPC = require("./rpc"); const HK = require("./hk-util.js"); +const Store = require("./storage/file"); +const BlobStore = require("./storage/blob"); + module.exports.create = function (config, cb) { const Log = config.log; + var WARN = function (e, output) { + if (e && output) { + Log.warn(e, { + output: output, + message: String(e), + stack: new Error(e).stack, + }); + } + }; Log.silly('HK_LOADING', 'LOADING HISTORY_KEEPER MODULE'); @@ -25,9 +37,63 @@ module.exports.create = function (config, cb) { channel_cache: {}, queueStorage: WriteQueue(), batchIndexReads: BatchRead("HK_GET_INDEX"), + + //historyKeeper: config.historyKeeper, + intervals: config.intervals || {}, + maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024), + Sessions: {}, + paths: {}, + //msgStore: config.store, + + pinStore: undefined, + pinnedPads: {}, + pinsLoaded: false, + pendingPinInquiries: {}, + pendingUnpins: {}, + pinWorkers: 5, + + limits: {}, + admins: [], + Log: Log, + WARN: WARN, + flushCache: config.flushCache, + adminEmail: config.adminEmail, + allowSubscriptions: config.allowSubscriptions, + myDomain: config.myDomain, + mySubdomain: config.mySubdomain, + customLimits: config.customLimits, + // FIXME this attribute isn't in the default conf + // but it is referenced in Quota + domain: config.domain + }; + + var paths = Env.paths; + + var keyOrDefaultString = function (key, def) { + return typeof(config[key]) === 'string'? config[key]: def; }; - config.historyKeeper = { + var pinPath = paths.pin = keyOrDefaultString('pinPath', './pins'); + paths.block = keyOrDefaultString('blockPath', './block'); + paths.data = keyOrDefaultString('filePath', './datastore'); + paths.staging = keyOrDefaultString('blobStagingPath', './blobstage'); + paths.blob = keyOrDefaultString('blobPath', './blob'); + + Env.defaultStorageLimit = typeof(config.defaultStorageLimit) === 'number' && config.defaultStorageLimit > 0? + config.defaultStorageLimit: + Core.DEFAULT_LIMIT; + + try { + Env.admins = (config.adminKeys || []).map(function (k) { + k = k.replace(/\/+$/, ''); + var s = k.split('/'); + return s[s.length-1]; + }); + } catch (e) { + console.error("Can't parse admin keys. Please update or fix your config.js file!"); + } + + config.historyKeeper = Env.historyKeeper = { metadata_cache: Env.metadata_cache, channel_cache: Env.channel_cache, @@ -63,11 +129,34 @@ module.exports.create = function (config, cb) { Log.verbose('HK_ID', 'History keeper ID: ' + Env.id); nThen(function (w) { - require('./storage/file').create(config, w(function (_store) { + // create a pin store + Store.create({ + filePath: pinPath, + }, w(function (s) { + Env.pinStore = s; + })); + + // create a channel store + Store.create(config, w(function (_store) { config.store = _store; - Env.store = _store; + Env.msgStore = _store; // API used by rpc + Env.store = _store; // API used by historyKeeper + })); + + // create a blob store + BlobStore.create({ + blobPath: config.blobPath, + blobStagingPath: config.blobStagingPath, + archivePath: config.archivePath, + getSession: function (safeKey) { + return Core.getSession(Sessions, safeKey); + }, + }, w(function (err, blob) { + if (err) { throw new Error(err); } + Env.blobStore = blob; })); }).nThen(function (w) { + // create a task store require("./storage/tasks").create(config, w(function (e, tasks) { if (e) { throw e; @@ -87,7 +176,7 @@ module.exports.create = function (config, cb) { }, 1000 * 60 * 5); // run every five minutes })); }).nThen(function () { - RPC.create(config, function (err, _rpc) { + RPC.create(Env, function (err, _rpc) { if (err) { throw err; } Env.rpc = _rpc; diff --git a/lib/rpc.js b/lib/rpc.js index b5c142af5..875032b74 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -1,6 +1,4 @@ /*jshint esversion: 6 */ -const nThen = require("nthen"); - const Util = require("./common-util"); const Core = require("./commands/core"); @@ -14,9 +12,6 @@ const Upload = require("./commands/upload"); var RPC = module.exports; -const Store = require("./storage/file"); -const BlobStore = require("./storage/blob"); - const UNAUTHENTICATED_CALLS = { GET_FILE_SIZE: Pinning.getFileSize, GET_MULTIPLE_FILE_SIZE: Pinning.getMultipleFileSize, @@ -187,86 +182,12 @@ var rpc = function (Env, Server, data, respond) { return void respond("INVALID_RPC_CALL"); }; -RPC.create = function (config, cb) { - var Log = config.log; - - // load pin-store... - Log.silly('LOADING RPC MODULE'); - - var keyOrDefaultString = function (key, def) { - return typeof(config[key]) === 'string'? config[key]: def; - }; - - var WARN = function (e, output) { - if (e && output) { - Log.warn(e, { - output: output, - message: String(e), - stack: new Error(e).stack, - }); - } - }; - - if (typeof(config.domain) !== 'undefined') { - throw new Error('fuck'); - } - - var Env = { - historyKeeper: config.historyKeeper, - intervals: config.intervals || {}, - maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024), - Sessions: {}, - paths: {}, - msgStore: config.store, - - pinStore: undefined, - pinnedPads: {}, - pinsLoaded: false, - pendingPinInquiries: {}, - pendingUnpins: {}, - pinWorkers: 5, - - limits: {}, - admins: [], - Log: Log, - WARN: WARN, - flushCache: config.flushCache, - adminEmail: config.adminEmail, - allowSubscriptions: config.allowSubscriptions, - myDomain: config.myDomain, - mySubdomain: config.mySubdomain, - customLimits: config.customLimits, - // FIXME this attribute isn't in the default conf - // but it is referenced in Quota - domain: config.domain - }; - - Env.defaultStorageLimit = typeof(config.defaultStorageLimit) === 'number' && config.defaultStorageLimit > 0? - config.defaultStorageLimit: - Core.DEFAULT_LIMIT; - - try { - Env.admins = (config.adminKeys || []).map(function (k) { - k = k.replace(/\/+$/, ''); - var s = k.split('/'); - return s[s.length-1]; - }); - } catch (e) { - console.error("Can't parse admin keys. Please update or fix your config.js file!"); - } - +RPC.create = function (Env, cb) { var Sessions = Env.Sessions; - var paths = Env.paths; - var pinPath = paths.pin = keyOrDefaultString('pinPath', './pins'); - paths.block = keyOrDefaultString('blockPath', './block'); - paths.data = keyOrDefaultString('filePath', './datastore'); - paths.staging = keyOrDefaultString('blobStagingPath', './blobstage'); - paths.blob = keyOrDefaultString('blobPath', './blob'); - var updateLimitDaily = function () { Quota.updateCachedLimits(Env, function (e) { if (e) { - WARN('limitUpdate', e); + Env.WARN('limitUpdate', e); } }); }; @@ -276,35 +197,17 @@ RPC.create = function (config, cb) { Pinning.loadChannelPins(Env); - nThen(function (w) { - Store.create({ - filePath: pinPath, - }, w(function (s) { - Env.pinStore = s; - })); - BlobStore.create({ - blobPath: config.blobPath, - blobStagingPath: config.blobStagingPath, - archivePath: config.archivePath, - getSession: function (safeKey) { - return Core.getSession(Sessions, safeKey); - }, - }, w(function (err, blob) { - if (err) { throw new Error(err); } - Env.blobStore = blob; - })); - }).nThen(function () { - cb(void 0, function (Server, data, respond) { - try { - return rpc(Env, Server, data, respond); - } catch (e) { - console.log("Error from RPC with data " + JSON.stringify(data)); - console.log(e.stack); - } - }); - // expire old sessions once per minute - Env.intervals.sessionExpirationInterval = setInterval(function () { - Core.expireSessions(Sessions); - }, Core.SESSION_EXPIRATION_TIME); + // expire old sessions once per minute + Env.intervals.sessionExpirationInterval = setInterval(function () { + Core.expireSessions(Sessions); + }, Core.SESSION_EXPIRATION_TIME); + + cb(void 0, function (Server, data, respond) { + try { + return rpc(Env, Server, data, respond); + } catch (e) { + console.log("Error from RPC with data " + JSON.stringify(data)); + console.log(e.stack); + } }); }; From 8694c1702369f1710541d19726251257a541b84c Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 09:26:36 -0500 Subject: [PATCH 05/14] sketch out metadata commands for allow lists --- lib/metadata.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/metadata.js b/lib/metadata.js index 220327456..92a5d7541 100644 --- a/lib/metadata.js +++ b/lib/metadata.js @@ -19,6 +19,19 @@ var isValidOwner = function (owner) { return typeof(owner) === 'string' && owner.length === 44; }; +// ["RESTRICT_ACCESS", [true], 1561623438989] +// ["RESTRICT_ACCESS", [false], 1561623438989] +// commands.RESTRICT_ACCESS = function (meta, args) {}; + +// ["ADD_ALLOWED", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I=", ...], 1561623438989] +// commands.ADD_ALLOWED = function (meta, args) {}; + +// ["RM_ALLOWED", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I=", ...], 1561623438989] +// commands.RM_ALLOWED = function (meta, args) {}; + +// ["RESET_ALLOWED", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I=", ...], 1561623438989] +// commands.RESET_ALLOWED = function (meta, args) {}; + // ["ADD_OWNERS", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I="], 1561623438989] commands.ADD_OWNERS = function (meta, args) { // bail out if args isn't an array From 5dff6535ed3f748d133d96090f295255e3be9a44 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 10:19:51 -0500 Subject: [PATCH 06/14] add a simple guard against unparsed messages when trimming history --- lib/storage/file.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/storage/file.js b/lib/storage/file.js index 6b1577d80..1d4061c4b 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -970,6 +970,15 @@ var trimChannel = function (env, channelName, hash, _cb) { } var msg = Util.tryParse(s_msg); + + if (!msg) { + Log.error("TRIM_HISTORY_UNPARSED_LINE", { + content: s_msg, + index: i, + }); + return void readMore(); + } + var msgHash = Extras.getHash(msg[4]); if (msgHash === hash) { From e8949168ecd668d3b1ca024962c36affdf0e2df2 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 10:30:44 -0500 Subject: [PATCH 07/14] lint compliance --- lib/historyKeeper.js | 4 ++-- lib/storage/file.js | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index e99b279eb..49c199141 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -6,6 +6,7 @@ const WriteQueue = require("./write-queue"); const BatchRead = require("./batch-read"); const RPC = require("./rpc"); const HK = require("./hk-util.js"); +const Core = require("./commands/core"); const Store = require("./storage/file"); const BlobStore = require("./storage/blob"); @@ -54,7 +55,6 @@ module.exports.create = function (config, cb) { limits: {}, admins: [], - Log: Log, WARN: WARN, flushCache: config.flushCache, adminEmail: config.adminEmail, @@ -149,7 +149,7 @@ module.exports.create = function (config, cb) { blobStagingPath: config.blobStagingPath, archivePath: config.archivePath, getSession: function (safeKey) { - return Core.getSession(Sessions, safeKey); + return Core.getSession(Env.Sessions, safeKey); }, }, w(function (err, blob) { if (err) { throw new Error(err); } diff --git a/lib/storage/file.js b/lib/storage/file.js index 1d4061c4b..ee9a4fd91 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -970,14 +970,7 @@ var trimChannel = function (env, channelName, hash, _cb) { } var msg = Util.tryParse(s_msg); - - if (!msg) { - Log.error("TRIM_HISTORY_UNPARSED_LINE", { - content: s_msg, - index: i, - }); - return void readMore(); - } + if (!msg) { return void readMore(); } var msgHash = Extras.getHash(msg[4]); From b56367414b085131710f37b8ca1489dda9cd2603 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 11:31:00 -0500 Subject: [PATCH 08/14] don't overwrite cached indices when new users join a channel --- lib/historyKeeper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 03f31383b..348e3cc3f 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -45,7 +45,7 @@ module.exports.create = function (config, cb) { HK.dropChannel(Env, channelName); }, channelOpen: function (Server, channelName, userId) { - Env.channel_cache[channelName] = {}; + Env.channel_cache[channelName] = Env.channel_cache[channelName] || {}; Server.send(userId, [ 0, Env.id, From a4be6185de7eb80980176541fbd28cb50ad786d9 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 11:54:27 -0500 Subject: [PATCH 09/14] merge staging and do a little lint compliance --- lib/historyKeeper.js | 9 +++++++++ lib/hk-util.js | 6 +++--- lib/metadata.js | 41 +++++++++++++++++++++++++++++++++++++---- lib/rpc.js | 4 ++-- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index bccfef38f..a77a95740 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -112,6 +112,15 @@ module.exports.create = function (config, cb) { }, channelOpen: function (Server, channelName, userId) { Env.channel_cache[channelName] = Env.channel_cache[channelName] || {}; + + + // XXX RESTRICT + // this event is emitted whenever a user joins a channel. + // if that channel is restricted then we should forcefully disconnect them. + // we won't know that it's restricted until we load its metadata. + // as long as metadata is in memory as long as anyone is sending messages to a channel + // then we won't broadcast messages to unauthorized users + Server.send(userId, [ 0, Env.id, diff --git a/lib/hk-util.js b/lib/hk-util.js index 3221d50a2..3c5fe887c 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -789,9 +789,9 @@ const handleGetFullHistory = function (Env, Server, seq, userId, parsed) { }; const directMessageCommands = { - GET_HISTORY: handleGetHistory, - GET_HISTORY_RANGE: handleGetHistoryRange, - GET_FULL_HISTORY: handleGetFullHistory, + GET_HISTORY: handleGetHistory, // XXX RESTRICT + GET_HISTORY_RANGE: handleGetHistoryRange, // XXX RESTRICT + GET_FULL_HISTORY: handleGetFullHistory, // XXX RESTRICT }; /* onDirectMessage diff --git a/lib/metadata.js b/lib/metadata.js index 92a5d7541..aa5b49517 100644 --- a/lib/metadata.js +++ b/lib/metadata.js @@ -21,16 +21,49 @@ var isValidOwner = function (owner) { // ["RESTRICT_ACCESS", [true], 1561623438989] // ["RESTRICT_ACCESS", [false], 1561623438989] -// commands.RESTRICT_ACCESS = function (meta, args) {}; +commands.RESTRICT_ACCESS = function (meta, args) { + if (!Array.isArray(args) || typeof(args[0]) !== 'boolean') { + throw new Error('INVALID_STATE'); + } + + var bool = args[0]; + + // reject the proposed command if there is no change in state + if (meta.restricted === bool) { return false; } + + // apply the new state + meta.restricted = args[0]; + + // if you're disabling access restrictions then you can assume + // then there is nothing more to do. Leave the existing list as-is + if (!bool) { return true; } + + // you're all set if an allow list already exists + if (Array.isArray(meta.allowed)) { return true; } + + // otherwise define it + meta.allowed = []; + + return true; +}; // ["ADD_ALLOWED", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I=", ...], 1561623438989] -// commands.ADD_ALLOWED = function (meta, args) {}; +commands.ADD_ALLOWED = function (meta, args) { + args = args; + throw new Error('NOT_IMPLEMENTED'); +}; // ["RM_ALLOWED", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I=", ...], 1561623438989] -// commands.RM_ALLOWED = function (meta, args) {}; +commands.RM_ALLOWED = function (meta, args) { + args = args; + throw new Error('NOT_IMPLEMENTED'); +}; // ["RESET_ALLOWED", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I=", ...], 1561623438989] -// commands.RESET_ALLOWED = function (meta, args) {}; +commands.RESET_ALLOWED = function (meta, args) { + args = args; + throw new Error('NOT_IMPLEMENTED'); +}; // ["ADD_OWNERS", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I="], 1561623438989] commands.ADD_OWNERS = function (meta, args) { diff --git a/lib/rpc.js b/lib/rpc.js index 875032b74..0d9c605f7 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -18,8 +18,8 @@ const UNAUTHENTICATED_CALLS = { GET_DELETED_PADS: Pinning.getDeletedPads, IS_CHANNEL_PINNED: Pinning.isChannelPinned, IS_NEW_CHANNEL: Channel.isNewChannel, - WRITE_PRIVATE_MESSAGE: Channel.writePrivateMessage, - GET_METADATA: Metadata.getMetadata, + WRITE_PRIVATE_MESSAGE: Channel.writePrivateMessage, // XXX RESTRICT + GET_METADATA: Metadata.getMetadata, // XXX RESTRICT }; var isUnauthenticateMessage = function (msg) { From ff73e96cb80a9ca8a6dc87bbeb7c8207de5d7372 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 12:02:49 -0500 Subject: [PATCH 10/14] reimplement the trim history fix from staging --- lib/storage/file.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/storage/file.js b/lib/storage/file.js index 6b1577d80..b1ac4de3d 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -970,6 +970,7 @@ var trimChannel = function (env, channelName, hash, _cb) { } var msg = Util.tryParse(s_msg); + if (!msg) { return void readMore(); } var msgHash = Extras.getHash(msg[4]); if (msgHash === hash) { From 156c37942dec7fbca62690f76df5259faf79425e Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 18 Feb 2020 14:16:07 -0500 Subject: [PATCH 11/14] implement new metadata commands related to allow lists and mailboxes --- lib/metadata.js | 150 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 142 insertions(+), 8 deletions(-) diff --git a/lib/metadata.js b/lib/metadata.js index aa5b49517..a8ed86fd3 100644 --- a/lib/metadata.js +++ b/lib/metadata.js @@ -2,23 +2,48 @@ var Meta = module.exports; var deduplicate = require("./common-util").deduplicateString; -/* Metadata fields: +/* Metadata fields and the commands that can modify them + +we assume that these commands can only be performed +by owners or in some cases pending owners. Thus +the owners field is guaranteed to exist. * channel * validateKey * owners * ADD_OWNERS * RM_OWNERS + * RESET_OWNERS + * pending_owners + * ADD_PENDING_OWNERS + * RM_PENDING_OWNERS * expire + * UPDATE_EXPIRATION (NOT_IMPLEMENTED) + * restricted + * RESTRICT_ACCESS + * allowed + * ADD_ALLOWED + * RM_ALLOWED + * RESET_ALLOWED + * ADD_OWNERS + * RESET_OWNERS + * mailbox + * ADD_MAILBOX + * RM_MAILBOX */ var commands = {}; -var isValidOwner = function (owner) { +var isValidPublicKey = function (owner) { return typeof(owner) === 'string' && owner.length === 44; }; +// isValidPublicKey is a better indication of what the above function does +// I'm preserving this function name in case we ever want to expand its +// criteria at a later time... +var isValidOwner = isValidPublicKey; + // ["RESTRICT_ACCESS", [true], 1561623438989] // ["RESTRICT_ACCESS", [false], 1561623438989] commands.RESTRICT_ACCESS = function (meta, args) { @@ -49,20 +74,95 @@ commands.RESTRICT_ACCESS = function (meta, args) { // ["ADD_ALLOWED", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I=", ...], 1561623438989] commands.ADD_ALLOWED = function (meta, args) { - args = args; - throw new Error('NOT_IMPLEMENTED'); + if (!Array.isArray(args)) { + throw new Error("INVALID_ARGS"); + } + + var allowed = meta.allowed || []; + + var changed = false; + args.forEach(function (arg) { + // don't add invalid public keys + if (!isValidPublicKey(arg)) { return; } + // don't add owners to the allow list + if (meta.owners.indexOf(arg) >= 0) { return; } + // don't duplicate entries in the allow list + if (allowed.indexOf(arg) >= 0) { return; } + allowed.push(arg); + changed = true; + }); + + if (changed) { + meta.allowed = meta.allowed || allowed; + } + + return changed; }; // ["RM_ALLOWED", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I=", ...], 1561623438989] commands.RM_ALLOWED = function (meta, args) { - args = args; - throw new Error('NOT_IMPLEMENTED'); + if (!Array.isArray(args)) { + throw new Error("INVALID_ARGS"); + } + + // there may not be anything to remove + if (!meta.allowed) { return false; } + + var changed = false; + args.forEach(function (arg) { + var index = meta.allowed.indexOf(arg); + if (index < 0) { return; } + meta.allowed.splice(index, 1); + changed = true; + }); + + return changed; +}; + +var arrayHasChanged = function (A, B) { + var changed; + A.some(function (a) { + if (B.indexOf(a) < 0) { return (changed = true); } + }); + if (changed) { return true; } + B.some(function (b) { + if (A.indexOf(b) < 0) { return (changed = true); } + }); + return changed; +}; + +var filterInPlace = function (A, f) { + for (var i = A.length - 1; i >= 0; i--) { + if (f(A[i], i, A)) { A.splice(i, 1); } + } }; // ["RESET_ALLOWED", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I=", ...], 1561623438989] commands.RESET_ALLOWED = function (meta, args) { - args = args; - throw new Error('NOT_IMPLEMENTED'); + if (!Array.isArray(args)) { throw new Error("INVALID_ARGS"); } + + var updated = args.filter(function (arg) { + // don't allow invalid public keys + if (!isValidPublicKey(arg)) { return false; } + // don't ever add owners to the allow list + if (meta.owners.indexOf(arg)) { return false; } + return true; + }); + + // this is strictly an optimization... + // a change in length is a clear indicator of a functional change + if (meta.allowed && meta.allowed.length !== updated.length) { + meta.allowed = updated; + return true; + } + + // otherwise we must check that the arrays contain distinct elements + // if there is no functional change, then return false + if (!arrayHasChanged(meta.allowed, updated)) { return false; } + + // otherwise overwrite the in-memory data and indicate that there was a change + meta.allowed = updated; + return true; }; // ["ADD_OWNERS", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I="], 1561623438989] @@ -86,6 +186,13 @@ commands.ADD_OWNERS = function (meta, args) { changed = true; }); + if (changed && Array.isArray(meta.allowed)) { + // make sure owners are not included in the allow list + filterInPlace(meta.allowed, function (member) { + return meta.owners.indexOf(member) !== -1; + }); + } + return changed; }; @@ -187,6 +294,14 @@ commands.RESET_OWNERS = function (meta, args) { // overwrite the existing owners with the new one meta.owners = deduplicate(args.filter(isValidOwner)); + + if (Array.isArray(meta.allowed)) { + // make sure owners are not included in the allow list + filterInPlace(meta.allowed, function (member) { + return meta.owners.indexOf(member) !== -1; + }); + } + return true; }; @@ -224,6 +339,25 @@ commands.ADD_MAILBOX = function (meta, args) { return changed; }; +commands.RM_MAILBOX = function (meta, args) { + if (!Array.isArray(args)) { throw new Error("INVALID_ARGS"); } + if (!meta.mailbox || typeof(meta.mailbox) === 'undefined') { + return false; + } + if (typeof(meta.mailbox) === 'string' && args.length === 0) { + delete meta.mailbox; + return true; + } + + var changed = false; + args.forEach(function (arg) { + if (meta.mailbox[arg] === 'undefined') { return; } + delete meta.mailbox[arg]; + changed = true; + }); + return changed; +}; + commands.UPDATE_EXPIRATION = function () { throw new Error("E_NOT_IMPLEMENTED"); }; From eac3e4cbcdf935a3450b77c3b34fc7dcb6dd0339 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 18 Feb 2020 16:05:12 -0500 Subject: [PATCH 12/14] sketch out some historyKeeper functionality related to access lists --- lib/historyKeeper.js | 4 ++++ lib/hk-util.js | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index a77a95740..1b306681e 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -113,6 +113,10 @@ module.exports.create = function (config, cb) { channelOpen: function (Server, channelName, userId) { Env.channel_cache[channelName] = Env.channel_cache[channelName] || {}; + //const metadata = Env.metadata_cache[channelName]; + // chainpad-server@4.0.3 supports a removeFromChannel method + // Server.removeFromChannel(channelName, userId); + // this lets us kick users from restricted channels // XXX RESTRICT // this event is emitted whenever a user joins a channel. diff --git a/lib/hk-util.js b/lib/hk-util.js index 3c5fe887c..c94a67bd5 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -75,6 +75,29 @@ const isMetadataMessage = function (parsed) { return Boolean(parsed && parsed.channel); }; +const isChannelRestricted = function (metadata) { // XXX RESTRICT + metadata = metadata; + return false; +}; + +const isUserAllowed = function (metadata, userId) { // XXX RESTRICT +/* + + at this point all we have is the user's netflux id. + the allow-list is encoded for 'unsafeKeys' (URL-unsafe base64 encoded public signing keys). + + we need a lookup table: netfluxId => public keys with which this netflux session has authenticated. + from there we can check whether the user has authenticated for any of the allowed keys this session. + + owners are implicitly allowed to view any file they own. + pending_owners too. + otherwise check metadata.allowed. + +*/ + userId = userId; + return false; +}; + // validateKeyStrings supplied by clients must decode to 32-byte Uint8Arrays const isValidValidateKeyString = function (key) { try { @@ -646,6 +669,16 @@ const handleGetHistory = function (Env, Server, seq, userId, parsed) { // And then check if the channel is expired. If it is, send the error and abort // FIXME this is hard to read because 'checkExpired' has side effects if (checkExpired(Env, Server, channelName)) { return void waitFor.abort(); } + + // XXX RESTRICT + // once we've loaded the metadata we can check whether the channel is restricted + // and notify the user if they're not included in the list + if (isChannelRestricted(index.metadata) && isUserAllowed(index.metadata, userId)) { + // XXX RESTRICT send a message indicating that they need to authenticate + // for a list of private keys... + return void waitFor.abort(); + } + // always send metadata with GET_HISTORY requests Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(index.metadata)], w); })); @@ -817,6 +850,10 @@ HK.onDirectMessage = function (Env, Server, seq, userId, json) { // have to abort later (once we know the expiration time) if (checkExpired(Env, Server, parsed[1])) { return; } + // XXX RESTRICT + // metadata might already be in memory. + // rejecting unauthorized users here is an optimization + // look up the appropriate command in the map of commands or fall back to RPC var command = directMessageCommands[parsed[0]] || handleRPC; From 505e383f9f44133e55cad2344ee731f28ccf0f47 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 18 Feb 2020 16:06:06 -0500 Subject: [PATCH 13/14] test restricted access from the command line --- scripts/tests/test-rpc.js | 135 +++++++++++++++++++++++++++++++++++++- 1 file changed, 134 insertions(+), 1 deletion(-) diff --git a/scripts/tests/test-rpc.js b/scripts/tests/test-rpc.js index 07f30bc46..e944a498b 100644 --- a/scripts/tests/test-rpc.js +++ b/scripts/tests/test-rpc.js @@ -357,9 +357,142 @@ nThen(function (w) { bob.name = 'bob'; //console.log("Initialized Bob"); })); +}).nThen(function (w) { + // restrict access to oscar's mailbox channel + oscar.rpc.send('SET_METADATA', { + command: 'RESTRICT_ACCESS', + channel: oscar.mailboxChannel, + value: [ true ] + }, w(function (err, response) { + if (err) { + return void console.log(err); + } + var metadata = response[0]; + if (!(metadata && metadata.restricted)) { + throw new Error("EXPECTED MAILBOX TO BE RESTRICTED"); + } + })); +}).nThen(function (w) { + // XXX RESTRICT GET_METADATA should fail because alice is not on the allow list + // expect INSUFFICIENT_PERMISSIONS + alice.anonRpc.send('GET_METADATA', oscar.mailboxChannel, w(function (err) { + if (!err) { + // XXX RESTRICT alice should not be permitted to read oscar's mailbox's metadata + } + })); +}).nThen(function (w) { + // add alice to oscar's mailbox's allow list for some reason + oscar.rpc.send('SET_METADATA', { + command: 'ADD_ALLOWED', + channel: oscar.mailboxChannel, + value: [ + alice.edKeys.edPublic + ] + }, w(function (err /*, metadata */) { + if (err) { + return void console.error(err); + } + //console.log('XXX', metadata); + })); +}).nThen(function (w) { + oscar.anonRpc.send('GET_METADATA', oscar.mailboxChannel, w(function (err, response) { + if (err) { + throw new Error("OSCAR SHOULD BE ABLE TO READ HIS OWN METADATA"); + } + var metadata = response && response[0]; + + if (!metadata) { + throw new Error("EXPECTED METADATA"); + } + + if (metadata.allowed[0] !== alice.edKeys.edPublic) { + throw new Error("EXPECTED ALICE TO BE ON ALLOW LIST"); + } + })); }).nThen(function () { - //setTimeout(w(), 500); + // XXX RESTRICT alice should now be able to read oscar's mailbox metadata +/* + alice.anonRpc.send('GET_METADATA', oscar.mailboxChannel, function (err, response) { + if (err) { + PROBLEM + } + }); +*/ +}).nThen(function (w) { + //throw new Error("boop"); + // add alice as an owner of oscar's mailbox for some reason + oscar.rpc.send('SET_METADATA', { + command: 'ADD_OWNERS', + channel: oscar.mailboxChannel, + value: [ + alice.edKeys.edPublic + ] + }, Util.mkTimeout(w(function (err) { + if (err === 'TIMEOUT') { + throw new Error(err); + } + if (err) { + throw new Error("ADD_OWNERS_FAILURE"); + } + }), 2000)); +}).nThen(function (w) { + // alice should now be able to read oscar's mailbox metadata + alice.anonRpc.send('GET_METADATA', oscar.mailboxChannel, w(function (err, response) { + if (err) { + throw new Error("EXPECTED ALICE TO BE ALLOWED TO READ OSCAR'S METADATA"); + } + var metadata = response && response[0]; + if (!metadata) { throw new Error("EXPECTED METADATA"); } + if (metadata.allowed.length !== 0) { + throw new Error("EXPECTED AN EMPTY ALLOW LIST"); + } + })); +}).nThen(function (w) { + // disable the access restrictionallow list + oscar.rpc.send('SET_METADATA', { + command: 'RESTRICT_ACCESS', + channel: oscar.mailboxChannel, + value: [ + false + ] + }, w(function (err) { + if (err) { + throw new Error("COULD_NOT_DISABLE_RESTRICTED_ACCESS"); + } + })); + // add alice to oscar's mailbox's allow list for some reason + oscar.rpc.send('SET_METADATA', { + command: 'ADD_ALLOWED', + channel: oscar.mailboxChannel, + value: [ + bob.edKeys.edPublic + ] + }, w(function (err) { + if (err) { + return void console.error(err); + } + })); +}).nThen(function (w) { + oscar.anonRpc.send('GET_METADATA', oscar.mailboxChannel, w(function (err, response) { + if (err) { + throw new Error("OSCAR SHOULD BE ABLE TO READ HIS OWN METADATA"); + } + var metadata = response && response[0]; + + if (!metadata) { + throw new Error("EXPECTED METADATA"); + } + + if (metadata.allowed[0] !== bob.edKeys.edPublic) { + throw new Error("EXPECTED ALICE TO BE ON ALLOW LIST"); + } + if (metadata.restricted) { + throw new Error("RESTRICTED_ACCESS_NOT_DISABLED"); + } + })); +}).nThen(function () { + //setTimeout(w(), 500); }).nThen(function (w) { // Alice loads the roster... var rosterKeys = Crypto.Team.deriveMemberKeys(sharedConfig.rosterSeed, alice.curveKeys); From 27e57e7af06bb6888ee52551ebd83ab2510ef038 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 18 Feb 2020 16:07:14 -0500 Subject: [PATCH 14/14] oups! fix a missing callback that's been here for a while... --- lib/commands/metadata.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/commands/metadata.js b/lib/commands/metadata.js index 511f963bf..5b5e28f7e 100644 --- a/lib/commands/metadata.js +++ b/lib/commands/metadata.js @@ -42,8 +42,8 @@ Data.setMetadata = function (Env, safeKey, data, cb, Server) { var channel = data.channel; var command = data.command; if (!channel || !Core.isValidId(channel)) { return void cb ('INVALID_CHAN'); } - if (!command || typeof (command) !== 'string') { return void cb ('INVALID_COMMAND'); } - if (Meta.commands.indexOf(command) === -1) { return void('UNSUPPORTED_COMMAND'); } + if (!command || typeof (command) !== 'string') { return void cb('INVALID_COMMAND'); } + if (Meta.commands.indexOf(command) === -1) { return void cb('UNSUPPORTED_COMMAND'); } queueMetadata(channel, function (next) { Data.getMetadata(Env, channel, function (err, metadata) {