From 1fc8c1de164683b543f4c9fed8cc5ae29f00859e Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 08:48:15 -0500 Subject: [PATCH 1/8] add missing connect-src directives to example nginx conf --- docs/example.nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/example.nginx.conf b/docs/example.nginx.conf index 6d45b8198..80c799f8c 100644 --- a/docs/example.nginx.conf +++ b/docs/example.nginx.conf @@ -72,7 +72,7 @@ server { set $styleSrc "'unsafe-inline' 'self' ${main_domain}"; # connect-src restricts URLs which can be loaded using script interfaces - set $connectSrc "'self' https://${main_domain} $main_domain https://${api_domain} blob:"; + set $connectSrc "'self' https://${main_domain} ${main_domain} https://${api_domain} blob: wss://${api_domain} ${api_domain} ${files_domain}"; # fonts can be loaded from data-URLs or the main domain set $fontSrc "'self' data: ${main_domain}"; From 8700345ccc0c9d69e96bff7cd5304a44b0d62011 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 08:48:15 -0500 Subject: [PATCH 2/8] add missing connect-src directives to example nginx conf --- docs/example.nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/example.nginx.conf b/docs/example.nginx.conf index 6d45b8198..80c799f8c 100644 --- a/docs/example.nginx.conf +++ b/docs/example.nginx.conf @@ -72,7 +72,7 @@ server { set $styleSrc "'unsafe-inline' 'self' ${main_domain}"; # connect-src restricts URLs which can be loaded using script interfaces - set $connectSrc "'self' https://${main_domain} $main_domain https://${api_domain} blob:"; + set $connectSrc "'self' https://${main_domain} ${main_domain} https://${api_domain} blob: wss://${api_domain} ${api_domain} ${files_domain}"; # fonts can be loaded from data-URLs or the main domain set $fontSrc "'self' data: ${main_domain}"; From f86196e40adfccade922e63bba5dafbff20538fe Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 09:01:10 -0500 Subject: [PATCH 3/8] implement shared environment between historyKeeper and RPC --- lib/commands/channel.js | 10 ++-- lib/commands/metadata.js | 4 +- lib/historyKeeper.js | 97 ++++++++++++++++++++++++++++-- lib/rpc.js | 125 +++++---------------------------------- 4 files changed, 114 insertions(+), 122 deletions(-) diff --git a/lib/commands/channel.js b/lib/commands/channel.js index 88404a9b2..6296aee0e 100644 --- a/lib/commands/channel.js +++ b/lib/commands/channel.js @@ -23,7 +23,7 @@ Channel.clearOwnedChannel = function (Env, safeKey, channelId, cb, Server) { if (e) { return void cb(e); } cb(); - const channel_cache = Env.historyKeeper.channel_cache; + const channel_cache = Env.channel_cache; const clear = function () { // delete the channel cache because it will have been invalidated @@ -117,8 +117,8 @@ Channel.removeOwnedChannel = function (Env, safeKey, channelId, cb, Server) { } cb(void 0, 'OK'); - const channel_cache = Env.historyKeeper.channel_cache; - const metadata_cache = Env.historyKeeper.metadata_cache; + const channel_cache = Env.channel_cache; + const metadata_cache = Env.metadata_cache; const clear = function () { delete channel_cache[channelId]; @@ -187,8 +187,8 @@ Channel.trimHistory = function (Env, safeKey, data, cb) { // clear historyKeeper's cache for this channel Env.historyKeeper.channelClose(channelId); cb(void 0, 'OK'); - delete Env.historyKeeper.channel_cache[channelId]; - delete Env.historyKeeper.metadata_cache[channelId]; + delete Env.channel_cache[channelId]; + delete Env.metadata_cache[channelId]; }); }); }; diff --git a/lib/commands/metadata.js b/lib/commands/metadata.js index 41aea9888..511f963bf 100644 --- a/lib/commands/metadata.js +++ b/lib/commands/metadata.js @@ -111,8 +111,8 @@ Data.setMetadata = function (Env, safeKey, data, cb, Server) { cb(void 0, metadata); next(); - const metadata_cache = Env.historyKeeper.metadata_cache; - const channel_cache = Env.historyKeeper.channel_cache; + const metadata_cache = Env.metadata_cache; + const channel_cache = Env.channel_cache; metadata_cache[channel] = metadata; diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 03f31383b..e99b279eb 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -7,8 +7,20 @@ const BatchRead = require("./batch-read"); const RPC = require("./rpc"); const HK = require("./hk-util.js"); +const Store = require("./storage/file"); +const BlobStore = require("./storage/blob"); + module.exports.create = function (config, cb) { const Log = config.log; + var WARN = function (e, output) { + if (e && output) { + Log.warn(e, { + output: output, + message: String(e), + stack: new Error(e).stack, + }); + } + }; Log.silly('HK_LOADING', 'LOADING HISTORY_KEEPER MODULE'); @@ -25,9 +37,63 @@ module.exports.create = function (config, cb) { channel_cache: {}, queueStorage: WriteQueue(), batchIndexReads: BatchRead("HK_GET_INDEX"), + + //historyKeeper: config.historyKeeper, + intervals: config.intervals || {}, + maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024), + Sessions: {}, + paths: {}, + //msgStore: config.store, + + pinStore: undefined, + pinnedPads: {}, + pinsLoaded: false, + pendingPinInquiries: {}, + pendingUnpins: {}, + pinWorkers: 5, + + limits: {}, + admins: [], + Log: Log, + WARN: WARN, + flushCache: config.flushCache, + adminEmail: config.adminEmail, + allowSubscriptions: config.allowSubscriptions, + myDomain: config.myDomain, + mySubdomain: config.mySubdomain, + customLimits: config.customLimits, + // FIXME this attribute isn't in the default conf + // but it is referenced in Quota + domain: config.domain + }; + + var paths = Env.paths; + + var keyOrDefaultString = function (key, def) { + return typeof(config[key]) === 'string'? config[key]: def; }; - config.historyKeeper = { + var pinPath = paths.pin = keyOrDefaultString('pinPath', './pins'); + paths.block = keyOrDefaultString('blockPath', './block'); + paths.data = keyOrDefaultString('filePath', './datastore'); + paths.staging = keyOrDefaultString('blobStagingPath', './blobstage'); + paths.blob = keyOrDefaultString('blobPath', './blob'); + + Env.defaultStorageLimit = typeof(config.defaultStorageLimit) === 'number' && config.defaultStorageLimit > 0? + config.defaultStorageLimit: + Core.DEFAULT_LIMIT; + + try { + Env.admins = (config.adminKeys || []).map(function (k) { + k = k.replace(/\/+$/, ''); + var s = k.split('/'); + return s[s.length-1]; + }); + } catch (e) { + console.error("Can't parse admin keys. Please update or fix your config.js file!"); + } + + config.historyKeeper = Env.historyKeeper = { metadata_cache: Env.metadata_cache, channel_cache: Env.channel_cache, @@ -63,11 +129,34 @@ module.exports.create = function (config, cb) { Log.verbose('HK_ID', 'History keeper ID: ' + Env.id); nThen(function (w) { - require('./storage/file').create(config, w(function (_store) { + // create a pin store + Store.create({ + filePath: pinPath, + }, w(function (s) { + Env.pinStore = s; + })); + + // create a channel store + Store.create(config, w(function (_store) { config.store = _store; - Env.store = _store; + Env.msgStore = _store; // API used by rpc + Env.store = _store; // API used by historyKeeper + })); + + // create a blob store + BlobStore.create({ + blobPath: config.blobPath, + blobStagingPath: config.blobStagingPath, + archivePath: config.archivePath, + getSession: function (safeKey) { + return Core.getSession(Sessions, safeKey); + }, + }, w(function (err, blob) { + if (err) { throw new Error(err); } + Env.blobStore = blob; })); }).nThen(function (w) { + // create a task store require("./storage/tasks").create(config, w(function (e, tasks) { if (e) { throw e; @@ -87,7 +176,7 @@ module.exports.create = function (config, cb) { }, 1000 * 60 * 5); // run every five minutes })); }).nThen(function () { - RPC.create(config, function (err, _rpc) { + RPC.create(Env, function (err, _rpc) { if (err) { throw err; } Env.rpc = _rpc; diff --git a/lib/rpc.js b/lib/rpc.js index b5c142af5..875032b74 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -1,6 +1,4 @@ /*jshint esversion: 6 */ -const nThen = require("nthen"); - const Util = require("./common-util"); const Core = require("./commands/core"); @@ -14,9 +12,6 @@ const Upload = require("./commands/upload"); var RPC = module.exports; -const Store = require("./storage/file"); -const BlobStore = require("./storage/blob"); - const UNAUTHENTICATED_CALLS = { GET_FILE_SIZE: Pinning.getFileSize, GET_MULTIPLE_FILE_SIZE: Pinning.getMultipleFileSize, @@ -187,86 +182,12 @@ var rpc = function (Env, Server, data, respond) { return void respond("INVALID_RPC_CALL"); }; -RPC.create = function (config, cb) { - var Log = config.log; - - // load pin-store... - Log.silly('LOADING RPC MODULE'); - - var keyOrDefaultString = function (key, def) { - return typeof(config[key]) === 'string'? config[key]: def; - }; - - var WARN = function (e, output) { - if (e && output) { - Log.warn(e, { - output: output, - message: String(e), - stack: new Error(e).stack, - }); - } - }; - - if (typeof(config.domain) !== 'undefined') { - throw new Error('fuck'); - } - - var Env = { - historyKeeper: config.historyKeeper, - intervals: config.intervals || {}, - maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024), - Sessions: {}, - paths: {}, - msgStore: config.store, - - pinStore: undefined, - pinnedPads: {}, - pinsLoaded: false, - pendingPinInquiries: {}, - pendingUnpins: {}, - pinWorkers: 5, - - limits: {}, - admins: [], - Log: Log, - WARN: WARN, - flushCache: config.flushCache, - adminEmail: config.adminEmail, - allowSubscriptions: config.allowSubscriptions, - myDomain: config.myDomain, - mySubdomain: config.mySubdomain, - customLimits: config.customLimits, - // FIXME this attribute isn't in the default conf - // but it is referenced in Quota - domain: config.domain - }; - - Env.defaultStorageLimit = typeof(config.defaultStorageLimit) === 'number' && config.defaultStorageLimit > 0? - config.defaultStorageLimit: - Core.DEFAULT_LIMIT; - - try { - Env.admins = (config.adminKeys || []).map(function (k) { - k = k.replace(/\/+$/, ''); - var s = k.split('/'); - return s[s.length-1]; - }); - } catch (e) { - console.error("Can't parse admin keys. Please update or fix your config.js file!"); - } - +RPC.create = function (Env, cb) { var Sessions = Env.Sessions; - var paths = Env.paths; - var pinPath = paths.pin = keyOrDefaultString('pinPath', './pins'); - paths.block = keyOrDefaultString('blockPath', './block'); - paths.data = keyOrDefaultString('filePath', './datastore'); - paths.staging = keyOrDefaultString('blobStagingPath', './blobstage'); - paths.blob = keyOrDefaultString('blobPath', './blob'); - var updateLimitDaily = function () { Quota.updateCachedLimits(Env, function (e) { if (e) { - WARN('limitUpdate', e); + Env.WARN('limitUpdate', e); } }); }; @@ -276,35 +197,17 @@ RPC.create = function (config, cb) { Pinning.loadChannelPins(Env); - nThen(function (w) { - Store.create({ - filePath: pinPath, - }, w(function (s) { - Env.pinStore = s; - })); - BlobStore.create({ - blobPath: config.blobPath, - blobStagingPath: config.blobStagingPath, - archivePath: config.archivePath, - getSession: function (safeKey) { - return Core.getSession(Sessions, safeKey); - }, - }, w(function (err, blob) { - if (err) { throw new Error(err); } - Env.blobStore = blob; - })); - }).nThen(function () { - cb(void 0, function (Server, data, respond) { - try { - return rpc(Env, Server, data, respond); - } catch (e) { - console.log("Error from RPC with data " + JSON.stringify(data)); - console.log(e.stack); - } - }); - // expire old sessions once per minute - Env.intervals.sessionExpirationInterval = setInterval(function () { - Core.expireSessions(Sessions); - }, Core.SESSION_EXPIRATION_TIME); + // expire old sessions once per minute + Env.intervals.sessionExpirationInterval = setInterval(function () { + Core.expireSessions(Sessions); + }, Core.SESSION_EXPIRATION_TIME); + + cb(void 0, function (Server, data, respond) { + try { + return rpc(Env, Server, data, respond); + } catch (e) { + console.log("Error from RPC with data " + JSON.stringify(data)); + console.log(e.stack); + } }); }; From 5dff6535ed3f748d133d96090f295255e3be9a44 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 10:19:51 -0500 Subject: [PATCH 4/8] add a simple guard against unparsed messages when trimming history --- lib/storage/file.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/storage/file.js b/lib/storage/file.js index 6b1577d80..1d4061c4b 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -970,6 +970,15 @@ var trimChannel = function (env, channelName, hash, _cb) { } var msg = Util.tryParse(s_msg); + + if (!msg) { + Log.error("TRIM_HISTORY_UNPARSED_LINE", { + content: s_msg, + index: i, + }); + return void readMore(); + } + var msgHash = Extras.getHash(msg[4]); if (msgHash === hash) { From e8949168ecd668d3b1ca024962c36affdf0e2df2 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 10:30:44 -0500 Subject: [PATCH 5/8] lint compliance --- lib/historyKeeper.js | 4 ++-- lib/storage/file.js | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index e99b279eb..49c199141 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -6,6 +6,7 @@ const WriteQueue = require("./write-queue"); const BatchRead = require("./batch-read"); const RPC = require("./rpc"); const HK = require("./hk-util.js"); +const Core = require("./commands/core"); const Store = require("./storage/file"); const BlobStore = require("./storage/blob"); @@ -54,7 +55,6 @@ module.exports.create = function (config, cb) { limits: {}, admins: [], - Log: Log, WARN: WARN, flushCache: config.flushCache, adminEmail: config.adminEmail, @@ -149,7 +149,7 @@ module.exports.create = function (config, cb) { blobStagingPath: config.blobStagingPath, archivePath: config.archivePath, getSession: function (safeKey) { - return Core.getSession(Sessions, safeKey); + return Core.getSession(Env.Sessions, safeKey); }, }, w(function (err, blob) { if (err) { throw new Error(err); } diff --git a/lib/storage/file.js b/lib/storage/file.js index 1d4061c4b..ee9a4fd91 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -970,14 +970,7 @@ var trimChannel = function (env, channelName, hash, _cb) { } var msg = Util.tryParse(s_msg); - - if (!msg) { - Log.error("TRIM_HISTORY_UNPARSED_LINE", { - content: s_msg, - index: i, - }); - return void readMore(); - } + if (!msg) { return void readMore(); } var msgHash = Extras.getHash(msg[4]); From b56367414b085131710f37b8ca1489dda9cd2603 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 11:31:00 -0500 Subject: [PATCH 6/8] don't overwrite cached indices when new users join a channel --- lib/historyKeeper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 03f31383b..348e3cc3f 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -45,7 +45,7 @@ module.exports.create = function (config, cb) { HK.dropChannel(Env, channelName); }, channelOpen: function (Server, channelName, userId) { - Env.channel_cache[channelName] = {}; + Env.channel_cache[channelName] = Env.channel_cache[channelName] || {}; Server.send(userId, [ 0, Env.id, From ff73e96cb80a9ca8a6dc87bbeb7c8207de5d7372 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 12:02:49 -0500 Subject: [PATCH 7/8] reimplement the trim history fix from staging --- lib/storage/file.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/storage/file.js b/lib/storage/file.js index 6b1577d80..b1ac4de3d 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -970,6 +970,7 @@ var trimChannel = function (env, channelName, hash, _cb) { } var msg = Util.tryParse(s_msg); + if (!msg) { return void readMore(); } var msgHash = Extras.getHash(msg[4]); if (msgHash === hash) { From ecce654ca66f60e43ad818f503a534a00de26b6b Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 17 Feb 2020 12:48:10 -0500 Subject: [PATCH 8/8] add 'resource:' to script-src to enable shared-worker debugging in firefox --- docs/example.nginx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/example.nginx.conf b/docs/example.nginx.conf index 80c799f8c..ea8224c14 100644 --- a/docs/example.nginx.conf +++ b/docs/example.nginx.conf @@ -96,7 +96,7 @@ server { set $workerSrc "https://${main_domain}"; # script-src specifies valid sources for javascript, including inline handlers - set $scriptSrc "'self' ${main_domain}"; + set $scriptSrc "'self' resource: ${main_domain}"; set $unsafe 0; # the following assets are loaded via the sandbox domain @@ -110,7 +110,7 @@ server { # privileged contexts allow a few more rights than unprivileged contexts, though limits are still applied if ($unsafe) { - set $scriptSrc "'self' 'unsafe-eval' 'unsafe-inline' ${main_domain}"; + set $scriptSrc "'self' 'unsafe-eval' 'unsafe-inline' resource: ${main_domain}"; } # Finally, set all the rules you composed above.