From 1ea00cfdfa67a798ad7d6dccc1439e31188b4355 Mon Sep 17 00:00:00 2001 From: yflory Date: Thu, 19 Mar 2020 11:45:17 +0100 Subject: [PATCH 01/10] Remove invalid 'DISCONNECTED' message in drive history mode --- www/common/drive-ui.js | 6 ++++-- www/drive/inner.js | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/www/common/drive-ui.js b/www/common/drive-ui.js index 50506f248..21e51510e 100644 --- a/www/common/drive-ui.js +++ b/www/common/drive-ui.js @@ -259,12 +259,14 @@ define([ }; // Handle disconnect/reconnect - var setEditable = function (state) { + var setEditable = function (state, isHistory) { if (APP.closed || !APP.$content || !$.contains(document.documentElement, APP.$content[0])) { return; } APP.editable = !APP.readOnly && state; if (!state) { APP.$content.addClass('cp-app-drive-readonly'); - $('#cp-app-drive-connection-state').show(); + if (!isHistory) { + $('#cp-app-drive-connection-state').show(); + } $('[draggable="true"]').attr('draggable', false); } else { diff --git a/www/drive/inner.js b/www/drive/inner.js index f5251f284..720ac562d 100644 --- a/www/drive/inner.js +++ b/www/drive/inner.js @@ -119,7 +119,7 @@ define([ var setHistory = function (bool, update) { history.isHistoryMode = bool; - setEditable(!bool); + setEditable(!bool, true); if (!bool && update) { history.onLeaveHistory(); } From 12520ecb539840844c895900c83b15b38db13f0f Mon Sep 17 00:00:00 2001 From: yflory Date: Thu, 19 Mar 2020 11:46:39 +0100 Subject: [PATCH 02/10] Remove unnecessary log --- www/common/userObject.js | 1 - 1 file changed, 1 deletion(-) diff --git a/www/common/userObject.js b/www/common/userObject.js index db8e687d3..b11bcf4b1 100644 --- a/www/common/userObject.js +++ b/www/common/userObject.js @@ -203,7 +203,6 @@ define([ var base = exp.getStructure(); return typeof (obj) === "object" && Object.keys(base).every(function (key) { - console.log(key, obj[key], type(obj[key])); return obj[key] && type(base[key]) === type(obj[key]); }); }; From d386e223e4babf1909c728944b94ca05af117949 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 19 Mar 2020 17:33:22 -0400 Subject: [PATCH 03/10] simplify open/close of writeStreams --- lib/storage/file.js | 139 ++++++++++++++++---------------------------- 1 file changed, 49 insertions(+), 90 deletions(-) diff --git a/lib/storage/file.js b/lib/storage/file.js index 4f7cf54a6..c868d46a6 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -11,6 +11,7 @@ var Meta = require("../metadata"); var Extras = require("../hk-util"); const readFileBin = require("../stream-file").readFileBin; +const BatchRead = require("../batch-read"); const Schedule = require("../schedule"); const isValidChannelId = function (id) { @@ -59,9 +60,9 @@ var channelExists = function (filepath, cb) { }; const destroyStream = function (stream) { - stream.close(); + try { stream.close(); } catch (err) { console.error(err); } setTimeout(function () { - try { stream.destroy(); } catch (err) { console.log(err); } + try { stream.destroy(); } catch (err) { console.error(err); } }, 5000); }; @@ -141,7 +142,6 @@ var closeChannel = function (env, channelName, cb) { destroyStream(stream, channelName); } delete env.channels[channelName]; - env.openFiles--; cb(); } catch (err) { cb(err); @@ -290,7 +290,7 @@ var writeMetadata = function (env, channelId, data, cb) { // check if a file exists at $path -var checkPath = function (path, callback) { +var checkPath = function (path, callback) { // callback's second arg is never used... Fs.stat(path, function (err) { if (!err) { callback(undefined, true); @@ -632,7 +632,7 @@ var unarchiveChannel = function (env, channelName, cb) { })); }); }; - +/* var flushUnusedChannels = function (env, cb, frame) { var currentTime = +new Date(); @@ -654,6 +654,7 @@ var flushUnusedChannels = function (env, cb, frame) { }); cb(); }; +*/ /* channelBytes calls back with an error or the size (in bytes) of a channel and its metadata @@ -686,106 +687,63 @@ var channelBytes = function (env, chanName, cb) { }); }; -/*:: -export type ChainPadServer_ChannelInternal_t = { - atime: number, - writeStream: typeof(process.stdout), - whenLoaded: ?Array<(err:?Error, chan:?ChainPadServer_ChannelInternal_t)=>void>, - onError: Array<(?Error)=>void>, - path: string -}; -*/ -var getChannel = function ( // XXX BatchRead - env, - id, - _callback /*:(err:?Error, chan:?ChainPadServer_ChannelInternal_t)=>void*/ -) { - //console.log("getting channel [%s]", id); - var callback = Util.once(Util.mkAsync(_callback)); +var getChannel = function (env, id, _callback) { + var cb = Util.once(Util.mkAsync(_callback)); + + // if the channel is in memory if (env.channels[id]) { var chan = env.channels[id]; - chan.atime = +new Date(); - if (chan.whenLoaded) { - chan.whenLoaded.push(callback); - } else { - callback(undefined, chan); - } - return; + // delay its pending close a little longer + chan.delayClose(); + // and return its writeStream + return void cb(void 0, chan); } - if (env.openFiles >= env.openFileLimit) { - // FIXME warn if this is the case? - // alternatively use graceful-fs to handle lots of concurrent reads - // if you're running out of open files, asynchronously clean up expired files - // do it on a shorter timeframe, though (half of normal) - setTimeout(function () { - //console.log("FLUSHING UNUSED CHANNELS"); - flushUnusedChannels(env, function () { - if (env.verbose) { - console.log("Approaching open file descriptor limit. Cleaning up"); + // otherwise you need to open it or wait until its pending open completes + return void env.batchGetChannel(id, cb, function (done) { + var path = mkPath(env, id); + var channel = { + onError: [], + }; + nThen(function (w) { + // create the path to the file if it doesn't exist + checkPath(path, w(function (err) { + if (err) { + w.abort(); + return void done(err); } - }, env.channelExpirationMs / 2); - }); - } - var path = mkPath(env, id); - var channel /*:ChainPadServer_ChannelInternal_t*/ = env.channels[id] = { - atime: +new Date(), - writeStream: (undefined /*:any*/), - whenLoaded: [ callback ], - onError: [ ], - path: path - }; - var complete = function (err) { - var whenLoaded = channel.whenLoaded; - // no guarantee stream.on('error') will not cause this to be called multiple times - if (!whenLoaded) { return; } - channel.whenLoaded = undefined; - if (err) { - delete env.channels[id]; - } - if (!channel.writeStream) { - throw new Error("getChannel() complete called without channel writeStream"); // XXX - } - whenLoaded.forEach(function (wl) { wl(err, (err) ? undefined : channel); }); - }; - var fileExists; - nThen(function (waitFor) { - checkPath(path, waitFor(function (err, exists) { - if (err) { - waitFor.abort(); - return void complete(err); - } - fileExists = exists; - })); - }).nThen(function (waitFor) { - var stream = channel.writeStream = Fs.createWriteStream(path, { flags: 'a' }); // XXX - env.openFiles++; - stream.on('open', waitFor()); - stream.on('error', function (err /*:?Error*/) { - env.openFiles--; - // this might be called after this nThen block closes. - if (channel.whenLoaded) { - complete(err); - } else { + })); + }).nThen(function (w) { + var stream = channel.writeStream = Fs.createWriteStream(path, { flags: 'a' }); + stream.on('open', w()); + stream.on('error', function (err) { + w.abort(); + // this might be called after this nThen block closes. channel.onError.forEach(function (handler) { handler(err); }); - } + }); + }).nThen(function () { + channel.delayClose = Util.throttle(function () { + delete env.channels[id]; + destroyStream(channel.writeStream, path); + //console.log("closing writestream"); + }, 30000); + channel.delayClose(); + env.channels[id] = channel; + done(void 0, channel); }); - }).nThen(function () { - complete(); }); }; // write a message to the disk as raw bytes const messageBin = (env, chanName, msgBin, cb) => { var complete = Util.once(cb); - getChannel(env, chanName, function (err, chan) { // XXX + getChannel(env, chanName, function (err, chan) { if (!chan) { return void complete(err); } chan.onError.push(complete); chan.writeStream.write(msgBin, function () { chan.onError.splice(chan.onError.indexOf(complete), 1); - chan.atime = +new Date(); // XXX we should just throttle closing, much simpler and less error prone complete(); }); }); @@ -999,8 +957,7 @@ module.exports.create = function (conf, cb) { channels: { }, channelExpirationMs: conf.channelExpirationMs || 30000, verbose: conf.verbose, - openFiles: 0, - openFileLimit: conf.openFileLimit || 2048, + batchGetChannel: BatchRead('store_batch_channel'), }; var it; @@ -1232,7 +1189,8 @@ module.exports.create = function (conf, cb) { }, // iterate over open channels and close any that are not active flushUnusedChannels: function (cb) { - flushUnusedChannels(env, cb); + cb("DEPRECATED"); + //flushUnusedChannels(env, cb); }, // write to a log file log: function (channelName, content, cb) { @@ -1247,7 +1205,8 @@ module.exports.create = function (conf, cb) { } }); }); + /* it = setInterval(function () { flushUnusedChannels(env, function () { }); - }, 5000); + }, 5000);*/ }; From a13561eb8d27293d1d677580dfa7302951817510 Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 20 Mar 2020 13:45:45 +0100 Subject: [PATCH 04/10] Fix checkboxes in the markdown renderer #511 --- www/common/diffMarked.js | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index 783aa78a9..73c801ab4 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -117,29 +117,30 @@ define([ // Tasks list var checkedTaskItemPtn = /^\s*(

)?\[[xX]\](<\/p>)?\s*/; var uncheckedTaskItemPtn = /^\s*(

)?\[ ?\](<\/p>)?\s*/; - var bogusCheckPtn = //; + var bogusCheckPtn = //; + var bogusUncheckPtn = //; renderer.listitem = function (text) { var isCheckedTaskItem = checkedTaskItemPtn.test(text); var isUncheckedTaskItem = uncheckedTaskItemPtn.test(text); - var hasBogusInput = bogusCheckPtn.test(text); + var hasBogusCheckedInput = bogusCheckPtn.test(text); + var hasBogusUncheckedInput = bogusUncheckPtn.test(text); + var isCheckbox = true; if (isCheckedTaskItem) { text = text.replace(checkedTaskItemPtn, '') + '\n'; - } - if (isUncheckedTaskItem) { + } else if (isUncheckedTaskItem) { text = text.replace(uncheckedTaskItemPtn, '') + '\n'; - } - if (!isCheckedTaskItem && !isUncheckedTaskItem && hasBogusInput) { - if (/checked/.test(text)) { - text = text.replace(bogusCheckPtn, + } else if (hasBogusCheckedInput) { + text = text.replace(bogusCheckPtn, '') + '\n'; - } else if (/disabled/.test(text)) { - text = text.replace(bogusCheckPtn, + } else if (hasBogusUncheckedInput) { + text = text.replace(bogusUncheckPtn, '') + '\n'; - } + } else { + isCheckbox = false; } - var cls = (isCheckedTaskItem || isUncheckedTaskItem || hasBogusInput) ? ' class="todo-list-item"' : ''; + var cls = (isCheckbox) ? ' class="todo-list-item"' : ''; return '' + text + '\n'; }; restrictedRenderer.listitem = function (text) { From bf2e5aeca109edfb4428aae334e4ce64f21caad4 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 20 Mar 2020 17:04:18 -0400 Subject: [PATCH 05/10] add an admin panel block to display the open file count --- www/admin/inner.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/www/admin/inner.js b/www/admin/inner.js index ad7083c6c..acc789784 100644 --- a/www/admin/inner.js +++ b/www/admin/inner.js @@ -43,6 +43,7 @@ define([ 'stats': [ 'cp-admin-active-sessions', 'cp-admin-active-pads', + 'cp-admin-open-files', 'cp-admin-registered', 'cp-admin-disk-usage', ], @@ -119,6 +120,17 @@ define([ }); return $div; }; + create['open-files'] = function () { + var key = 'open-files'; // XXX + var $div = makeBlock(key); + sFrameChan.query('Q_ADMIN_RPC', { + cmd: 'GET_FILE_DESCRIPTOR_COUNT', + }, function (e, data) { + console.log(e, data); + $div.append(h('pre', String(data))); + }); + return $div; + }; create['registered'] = function () { var key = 'registered'; var $div = makeBlock(key); From 8e8b3716903df6dbb2f8f6fd9638242ea4860909 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 23 Mar 2020 12:32:03 +0100 Subject: [PATCH 06/10] Fix issues with trim history in spreadsheets --- www/common/common-ui-elements.js | 13 ++++++++++++- www/common/drive-ui.js | 1 + www/common/onlyoffice/inner.js | 3 +++ www/common/onlyoffice/main.js | 1 + www/common/outer/onlyoffice.js | 9 +++++++++ 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index f8a3711c0..fefbba6e1 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -803,6 +803,17 @@ define([ var chan = [data.channel]; if (data.rtChannel) { chan.push(data.rtChannel); } if (data.lastVersion) { chan.push(Hash.hrefToHexChannelId(data.lastVersion)); } + var channels = chan.filter(function (c) { return c.length === 32; }).map(function (id) { + if (id === data.rtChannel && data.lastVersion && data.lastCpHash) { + return { + channel: id, + lastKnownHash: data.lastCpHash + }; + } + return { + channel: id + }; + }); var history = common.makeUniversal('history'); var trimChannels = []; NThen(function (waitFor) { @@ -819,7 +830,7 @@ define([ if (!owned) { return; } history.execCommand('GET_HISTORY_SIZE', { pad: true, - channels: chan.filter(function (c) { return c.length === 32; }), + channels: channels, teamId: typeof(owned) === "number" && owned }, waitFor(function (obj) { if (obj && obj.error) { return; } diff --git a/www/common/drive-ui.js b/www/common/drive-ui.js index 21e51510e..c039a86e3 100644 --- a/www/common/drive-ui.js +++ b/www/common/drive-ui.js @@ -3867,6 +3867,7 @@ define([ var opts = {}; opts.href = Hash.getRelativeHref(data.href || data.roHref); + opts.channel = data.channel; if (manager.isSharedFolder(el)) { var ro = folders[el] && folders[el].version >= 2; diff --git a/www/common/onlyoffice/inner.js b/www/common/onlyoffice/inner.js index 0ad29df6c..f9ed15f60 100644 --- a/www/common/onlyoffice/inner.js +++ b/www/common/onlyoffice/inner.js @@ -325,6 +325,7 @@ define([ body: $('body'), onUploaded: function (ev, data) { if (!data || !data.url) { return; } + data.hash = ev.hash; sframeChan.query('Q_OO_SAVE', data, function (err) { onUploaded(ev, data, err); }); @@ -1556,6 +1557,7 @@ define([ content = hjson.content || content; var newLatest = getLastCp(); sframeChan.query('Q_OO_SAVE', { + hash: newLatest.hash, url: newLatest.file }, function () { }); newDoc = !content.hashes || Object.keys(content.hashes).length === 0; @@ -1649,6 +1651,7 @@ define([ var newLatest = getLastCp(); if (newLatest.index > latest.index) { sframeChan.query('Q_OO_SAVE', { + hash: newLatest.hash, url: newLatest.file }, function () { }); } diff --git a/www/common/onlyoffice/main.js b/www/common/onlyoffice/main.js index 7007bd360..9096cd730 100644 --- a/www/common/onlyoffice/main.js +++ b/www/common/onlyoffice/main.js @@ -61,6 +61,7 @@ define([ if (e) { return void cb(e); } Cryptpad.setPadAttribute('lastVersion', data.url, cb); }); + Cryptpad.setPadAttribute('lastCpHash', data.hash, cb); }); sframeChan.on('Q_OO_OPENCHANNEL', function (data, cb) { Cryptpad.getPadAttribute('rtChannel', function (err, res) { diff --git a/www/common/outer/onlyoffice.js b/www/common/outer/onlyoffice.js index 968ea12c3..567bc180b 100644 --- a/www/common/outer/onlyoffice.js +++ b/www/common/outer/onlyoffice.js @@ -36,6 +36,7 @@ define([ return void cb(); } + var txid = Math.floor(Math.random() * 1000000); var onOpen = function (wc) { ctx.channels[channel] = ctx.channels[channel] || { @@ -91,6 +92,7 @@ define([ var hk = network.historyKeeper; var cfg = { + txid: txid, lastKnownHash: chan.lastKnownHash || chan.lastCpHash, metadata: { validateKey: obj.validateKey, @@ -121,6 +123,8 @@ define([ } catch (e) {} if (!parsed) { return; } + // If there is a txid, make sure it's ours or abort + if (parsed.txid && parsed.txid !== txid) { return; } // Keep only metadata messages for the current channel if (parsed.channel && parsed.channel !== channel) { return; } @@ -138,6 +142,11 @@ define([ } if (parsed.error && parsed.channel) { return; } + // If there is a txid, make sure it's ours or abort + if (Array.isArray(parsed) && parsed[0] && parsed[0] !== txid) { + return; + } + msg = parsed[4]; // Keep only the history for our channel From a4d1b47a4ca8036936bd2a441b1a177c8debfc3f Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 24 Mar 2020 12:40:13 +0100 Subject: [PATCH 07/10] Fix 'readonly' mode after disconnection in onlyoffice --- www/common/onlyoffice/inner.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/www/common/onlyoffice/inner.js b/www/common/onlyoffice/inner.js index f9ed15f60..5ac3da269 100644 --- a/www/common/onlyoffice/inner.js +++ b/www/common/onlyoffice/inner.js @@ -104,10 +104,14 @@ define([ return metadataMgr.getNetfluxId() + '-' + privateData.clientId; }; + var getEditor = function () { + return window.frames[0].editor || window.frames[0].editorCell; + }; + var setEditable = function (state) { $('#cp-app-oo-editor').find('#cp-app-oo-offline').remove(); try { - window.frames[0].editor.asc_setViewMode(!state); + getEditor().asc_setViewMode(!state); //window.frames[0].editor.setViewModeDisconnect(true); } catch (e) {} if (!state && !readOnly) { @@ -241,10 +245,6 @@ define([ cpIndex: 0 }; - var getEditor = function () { - return window.frames[0].editor || window.frames[0].editorCell; - }; - var getContent = function () { try { return getEditor().asc_nativeGetFile(); @@ -847,7 +847,7 @@ define([ "id": String(myOOId), //"c0c3bf82-20d7-4663-bf6d-7fa39c598b1d", "firstname": metadataMgr.getUserData().name || Messages.anonymous, }, - "mode": readOnly || lock ? "view" : "edit", + "mode": lock ? "view" : "edit", "lang": (navigator.language || navigator.userLanguage || '').slice(0,2) }, "events": { From 9aaddea1482dc73122c2643c21e250403c2789e8 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 24 Mar 2020 13:18:32 +0100 Subject: [PATCH 08/10] Add edPublic to user profiles --- www/common/outer/profile.js | 3 +++ www/profile/inner.js | 30 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/www/common/outer/profile.js b/www/common/outer/profile.js index dd56db6e9..733b2b4c4 100644 --- a/www/common/outer/profile.js +++ b/www/common/outer/profile.js @@ -54,6 +54,9 @@ define([ if (!lm.proxy.notifications) { lm.proxy.notifications = Util.find(ctx.store.proxy, ['mailboxes', 'notifications', 'channel']); } + if (!lm.proxy.edPublic) { + lm.proxy.edPublic = ctx.store.proxy.edPublic; + } if (ctx.onReadyHandlers.length) { ctx.onReadyHandlers.forEach(function (f) { try { diff --git a/www/profile/inner.js b/www/profile/inner.js index 9626c7734..5dfbaf491 100644 --- a/www/profile/inner.js +++ b/www/profile/inner.js @@ -457,6 +457,34 @@ define([ APP.editor.save(); }; + Messages.profile_copyKey = "Copy edPublic key"; // XXX + var addPublicKey = function ($container) { + if (!APP.readOnly) { return; } + + var $div = $(h('div.cp-sidebarlayout-element')).appendTo($container); + APP.$edPublic = $('