diff --git a/lib/storage/file.js b/lib/storage/file.js index 172ed8e1a..25ff021f0 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -11,6 +11,7 @@ var Meta = require("../metadata"); var Extras = require("../hk-util"); const readFileBin = require("../stream-file").readFileBin; +const BatchRead = require("../batch-read"); const Schedule = require("../schedule"); const isValidChannelId = function (id) { @@ -62,7 +63,7 @@ const destroyStream = function (stream) { if (!stream) { return; } try { stream.close(); } catch (err) { console.error(err); } setTimeout(function () { - try { stream.destroy(); } catch (err) { console.log(err); } + try { stream.destroy(); } catch (err) { console.error(err); } }, 5000); }; @@ -142,7 +143,6 @@ var closeChannel = function (env, channelName, cb) { destroyStream(stream, channelName); } delete env.channels[channelName]; - env.openFiles--; cb(); } catch (err) { cb(err); @@ -291,7 +291,7 @@ var writeMetadata = function (env, channelId, data, cb) { // check if a file exists at $path -var checkPath = function (path, callback) { +var checkPath = function (path, callback) { // callback's second arg is never used... Fs.stat(path, function (err) { if (!err) { callback(undefined, true); @@ -633,7 +633,7 @@ var unarchiveChannel = function (env, channelName, cb) { })); }); }; - +/* var flushUnusedChannels = function (env, cb, frame) { var currentTime = +new Date(); @@ -655,6 +655,7 @@ var flushUnusedChannels = function (env, cb, frame) { }); cb(); }; +*/ /* channelBytes calls back with an error or the size (in bytes) of a channel and its metadata @@ -687,106 +688,63 @@ var channelBytes = function (env, chanName, cb) { }); }; -/*:: -export type ChainPadServer_ChannelInternal_t = { - atime: number, - writeStream: typeof(process.stdout), - whenLoaded: ?Array<(err:?Error, chan:?ChainPadServer_ChannelInternal_t)=>void>, - onError: Array<(?Error)=>void>, - path: string -}; -*/ -var getChannel = function ( // XXX BatchRead - env, - id, - _callback /*:(err:?Error, chan:?ChainPadServer_ChannelInternal_t)=>void*/ -) { - //console.log("getting channel [%s]", id); - var callback = Util.once(Util.mkAsync(_callback)); +var getChannel = function (env, id, _callback) { + var cb = Util.once(Util.mkAsync(_callback)); + + // if the channel is in memory if (env.channels[id]) { var chan = env.channels[id]; - chan.atime = +new Date(); - if (chan.whenLoaded) { - chan.whenLoaded.push(callback); - } else { - callback(undefined, chan); - } - return; + // delay its pending close a little longer + chan.delayClose(); + // and return its writeStream + return void cb(void 0, chan); } - if (env.openFiles >= env.openFileLimit) { - // FIXME warn if this is the case? - // alternatively use graceful-fs to handle lots of concurrent reads - // if you're running out of open files, asynchronously clean up expired files - // do it on a shorter timeframe, though (half of normal) - setTimeout(function () { - //console.log("FLUSHING UNUSED CHANNELS"); - flushUnusedChannels(env, function () { - if (env.verbose) { - console.log("Approaching open file descriptor limit. Cleaning up"); + // otherwise you need to open it or wait until its pending open completes + return void env.batchGetChannel(id, cb, function (done) { + var path = mkPath(env, id); + var channel = { + onError: [], + }; + nThen(function (w) { + // create the path to the file if it doesn't exist + checkPath(path, w(function (err) { + if (err) { + w.abort(); + return void done(err); } - }, env.channelExpirationMs / 2); - }); - } - var path = mkPath(env, id); - var channel /*:ChainPadServer_ChannelInternal_t*/ = env.channels[id] = { - atime: +new Date(), - writeStream: (undefined /*:any*/), - whenLoaded: [ callback ], - onError: [ ], - path: path - }; - var complete = function (err) { - var whenLoaded = channel.whenLoaded; - // no guarantee stream.on('error') will not cause this to be called multiple times - if (!whenLoaded) { return; } - channel.whenLoaded = undefined; - if (err) { - delete env.channels[id]; - } - if (!channel.writeStream) { - throw new Error("getChannel() complete called without channel writeStream"); // XXX - } - whenLoaded.forEach(function (wl) { wl(err, (err) ? undefined : channel); }); - }; - var fileExists; - nThen(function (waitFor) { - checkPath(path, waitFor(function (err, exists) { - if (err) { - waitFor.abort(); - return void complete(err); - } - fileExists = exists; - })); - }).nThen(function (waitFor) { - var stream = channel.writeStream = Fs.createWriteStream(path, { flags: 'a' }); // XXX - env.openFiles++; - stream.on('open', waitFor()); - stream.on('error', function (err /*:?Error*/) { - env.openFiles--; - // this might be called after this nThen block closes. - if (channel.whenLoaded) { - complete(err); - } else { + })); + }).nThen(function (w) { + var stream = channel.writeStream = Fs.createWriteStream(path, { flags: 'a' }); + stream.on('open', w()); + stream.on('error', function (err) { + w.abort(); + // this might be called after this nThen block closes. channel.onError.forEach(function (handler) { handler(err); }); - } + }); + }).nThen(function () { + channel.delayClose = Util.throttle(function () { + delete env.channels[id]; + destroyStream(channel.writeStream, path); + //console.log("closing writestream"); + }, 30000); + channel.delayClose(); + env.channels[id] = channel; + done(void 0, channel); }); - }).nThen(function () { - complete(); }); }; // write a message to the disk as raw bytes const messageBin = (env, chanName, msgBin, cb) => { var complete = Util.once(cb); - getChannel(env, chanName, function (err, chan) { // XXX + getChannel(env, chanName, function (err, chan) { if (!chan) { return void complete(err); } chan.onError.push(complete); chan.writeStream.write(msgBin, function () { chan.onError.splice(chan.onError.indexOf(complete), 1); - chan.atime = +new Date(); // XXX we should just throttle closing, much simpler and less error prone complete(); }); }); @@ -1000,8 +958,7 @@ module.exports.create = function (conf, cb) { channels: { }, channelExpirationMs: conf.channelExpirationMs || 30000, verbose: conf.verbose, - openFiles: 0, - openFileLimit: conf.openFileLimit || 2048, + batchGetChannel: BatchRead('store_batch_channel'), }; var it; @@ -1240,7 +1197,8 @@ module.exports.create = function (conf, cb) { }, // iterate over open channels and close any that are not active flushUnusedChannels: function (cb) { - flushUnusedChannels(env, cb); + cb("DEPRECATED"); + //flushUnusedChannels(env, cb); }, // write to a log file log: function (channelName, content, cb) { @@ -1255,7 +1213,8 @@ module.exports.create = function (conf, cb) { } }); }); + /* it = setInterval(function () { flushUnusedChannels(env, function () { }); - }, 5000); + }, 5000);*/ }; diff --git a/www/admin/inner.js b/www/admin/inner.js index ad7083c6c..acc789784 100644 --- a/www/admin/inner.js +++ b/www/admin/inner.js @@ -43,6 +43,7 @@ define([ 'stats': [ 'cp-admin-active-sessions', 'cp-admin-active-pads', + 'cp-admin-open-files', 'cp-admin-registered', 'cp-admin-disk-usage', ], @@ -119,6 +120,17 @@ define([ }); return $div; }; + create['open-files'] = function () { + var key = 'open-files'; // XXX + var $div = makeBlock(key); + sFrameChan.query('Q_ADMIN_RPC', { + cmd: 'GET_FILE_DESCRIPTOR_COUNT', + }, function (e, data) { + console.log(e, data); + $div.append(h('pre', String(data))); + }); + return $div; + }; create['registered'] = function () { var key = 'registered'; var $div = makeBlock(key); diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index f8a3711c0..fefbba6e1 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -803,6 +803,17 @@ define([ var chan = [data.channel]; if (data.rtChannel) { chan.push(data.rtChannel); } if (data.lastVersion) { chan.push(Hash.hrefToHexChannelId(data.lastVersion)); } + var channels = chan.filter(function (c) { return c.length === 32; }).map(function (id) { + if (id === data.rtChannel && data.lastVersion && data.lastCpHash) { + return { + channel: id, + lastKnownHash: data.lastCpHash + }; + } + return { + channel: id + }; + }); var history = common.makeUniversal('history'); var trimChannels = []; NThen(function (waitFor) { @@ -819,7 +830,7 @@ define([ if (!owned) { return; } history.execCommand('GET_HISTORY_SIZE', { pad: true, - channels: chan.filter(function (c) { return c.length === 32; }), + channels: channels, teamId: typeof(owned) === "number" && owned }, waitFor(function (obj) { if (obj && obj.error) { return; } diff --git a/www/common/drive-ui.js b/www/common/drive-ui.js index 50506f248..c039a86e3 100644 --- a/www/common/drive-ui.js +++ b/www/common/drive-ui.js @@ -259,12 +259,14 @@ define([ }; // Handle disconnect/reconnect - var setEditable = function (state) { + var setEditable = function (state, isHistory) { if (APP.closed || !APP.$content || !$.contains(document.documentElement, APP.$content[0])) { return; } APP.editable = !APP.readOnly && state; if (!state) { APP.$content.addClass('cp-app-drive-readonly'); - $('#cp-app-drive-connection-state').show(); + if (!isHistory) { + $('#cp-app-drive-connection-state').show(); + } $('[draggable="true"]').attr('draggable', false); } else { @@ -3865,6 +3867,7 @@ define([ var opts = {}; opts.href = Hash.getRelativeHref(data.href || data.roHref); + opts.channel = data.channel; if (manager.isSharedFolder(el)) { var ro = folders[el] && folders[el].version >= 2; diff --git a/www/common/onlyoffice/inner.js b/www/common/onlyoffice/inner.js index 0ad29df6c..5ac3da269 100644 --- a/www/common/onlyoffice/inner.js +++ b/www/common/onlyoffice/inner.js @@ -104,10 +104,14 @@ define([ return metadataMgr.getNetfluxId() + '-' + privateData.clientId; }; + var getEditor = function () { + return window.frames[0].editor || window.frames[0].editorCell; + }; + var setEditable = function (state) { $('#cp-app-oo-editor').find('#cp-app-oo-offline').remove(); try { - window.frames[0].editor.asc_setViewMode(!state); + getEditor().asc_setViewMode(!state); //window.frames[0].editor.setViewModeDisconnect(true); } catch (e) {} if (!state && !readOnly) { @@ -241,10 +245,6 @@ define([ cpIndex: 0 }; - var getEditor = function () { - return window.frames[0].editor || window.frames[0].editorCell; - }; - var getContent = function () { try { return getEditor().asc_nativeGetFile(); @@ -325,6 +325,7 @@ define([ body: $('body'), onUploaded: function (ev, data) { if (!data || !data.url) { return; } + data.hash = ev.hash; sframeChan.query('Q_OO_SAVE', data, function (err) { onUploaded(ev, data, err); }); @@ -846,7 +847,7 @@ define([ "id": String(myOOId), //"c0c3bf82-20d7-4663-bf6d-7fa39c598b1d", "firstname": metadataMgr.getUserData().name || Messages.anonymous, }, - "mode": readOnly || lock ? "view" : "edit", + "mode": lock ? "view" : "edit", "lang": (navigator.language || navigator.userLanguage || '').slice(0,2) }, "events": { @@ -1556,6 +1557,7 @@ define([ content = hjson.content || content; var newLatest = getLastCp(); sframeChan.query('Q_OO_SAVE', { + hash: newLatest.hash, url: newLatest.file }, function () { }); newDoc = !content.hashes || Object.keys(content.hashes).length === 0; @@ -1649,6 +1651,7 @@ define([ var newLatest = getLastCp(); if (newLatest.index > latest.index) { sframeChan.query('Q_OO_SAVE', { + hash: newLatest.hash, url: newLatest.file }, function () { }); } diff --git a/www/common/onlyoffice/main.js b/www/common/onlyoffice/main.js index 7007bd360..9096cd730 100644 --- a/www/common/onlyoffice/main.js +++ b/www/common/onlyoffice/main.js @@ -61,6 +61,7 @@ define([ if (e) { return void cb(e); } Cryptpad.setPadAttribute('lastVersion', data.url, cb); }); + Cryptpad.setPadAttribute('lastCpHash', data.hash, cb); }); sframeChan.on('Q_OO_OPENCHANNEL', function (data, cb) { Cryptpad.getPadAttribute('rtChannel', function (err, res) { diff --git a/www/common/outer/onlyoffice.js b/www/common/outer/onlyoffice.js index 968ea12c3..567bc180b 100644 --- a/www/common/outer/onlyoffice.js +++ b/www/common/outer/onlyoffice.js @@ -36,6 +36,7 @@ define([ return void cb(); } + var txid = Math.floor(Math.random() * 1000000); var onOpen = function (wc) { ctx.channels[channel] = ctx.channels[channel] || { @@ -91,6 +92,7 @@ define([ var hk = network.historyKeeper; var cfg = { + txid: txid, lastKnownHash: chan.lastKnownHash || chan.lastCpHash, metadata: { validateKey: obj.validateKey, @@ -121,6 +123,8 @@ define([ } catch (e) {} if (!parsed) { return; } + // If there is a txid, make sure it's ours or abort + if (parsed.txid && parsed.txid !== txid) { return; } // Keep only metadata messages for the current channel if (parsed.channel && parsed.channel !== channel) { return; } @@ -138,6 +142,11 @@ define([ } if (parsed.error && parsed.channel) { return; } + // If there is a txid, make sure it's ours or abort + if (Array.isArray(parsed) && parsed[0] && parsed[0] !== txid) { + return; + } + msg = parsed[4]; // Keep only the history for our channel diff --git a/www/common/outer/profile.js b/www/common/outer/profile.js index dd56db6e9..733b2b4c4 100644 --- a/www/common/outer/profile.js +++ b/www/common/outer/profile.js @@ -54,6 +54,9 @@ define([ if (!lm.proxy.notifications) { lm.proxy.notifications = Util.find(ctx.store.proxy, ['mailboxes', 'notifications', 'channel']); } + if (!lm.proxy.edPublic) { + lm.proxy.edPublic = ctx.store.proxy.edPublic; + } if (ctx.onReadyHandlers.length) { ctx.onReadyHandlers.forEach(function (f) { try { diff --git a/www/common/userObject.js b/www/common/userObject.js index db8e687d3..b11bcf4b1 100644 --- a/www/common/userObject.js +++ b/www/common/userObject.js @@ -203,7 +203,6 @@ define([ var base = exp.getStructure(); return typeof (obj) === "object" && Object.keys(base).every(function (key) { - console.log(key, obj[key], type(obj[key])); return obj[key] && type(base[key]) === type(obj[key]); }); }; diff --git a/www/drive/inner.js b/www/drive/inner.js index f5251f284..720ac562d 100644 --- a/www/drive/inner.js +++ b/www/drive/inner.js @@ -119,7 +119,7 @@ define([ var setHistory = function (bool, update) { history.isHistoryMode = bool; - setEditable(!bool); + setEditable(!bool, true); if (!bool && update) { history.onLeaveHistory(); } diff --git a/www/profile/inner.js b/www/profile/inner.js index 9626c7734..5dfbaf491 100644 --- a/www/profile/inner.js +++ b/www/profile/inner.js @@ -457,6 +457,34 @@ define([ APP.editor.save(); }; + Messages.profile_copyKey = "Copy edPublic key"; // XXX + var addPublicKey = function ($container) { + if (!APP.readOnly) { return; } + + var $div = $(h('div.cp-sidebarlayout-element')).appendTo($container); + APP.$edPublic = $('