From a66d8c13841e526ecd086572365adc7dc5048de9 Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 18 Jan 2019 18:17:34 +0100 Subject: [PATCH] Use lastKnownHash to handle checkpoints in the realtime channel --- www/common/onlyoffice/inner.js | 281 ++++++++++++++++++++---------- www/common/onlyoffice/main.js | 20 ++- www/common/outer/async-store.js | 1 + www/common/outer/onlyoffice.js | 69 ++++---- www/common/sframe-common-outer.js | 1 - 5 files changed, 234 insertions(+), 138 deletions(-) diff --git a/www/common/onlyoffice/inner.js b/www/common/onlyoffice/inner.js index bfbecbff1..e5d1692ca 100644 --- a/www/common/onlyoffice/inner.js +++ b/www/common/onlyoffice/inner.js @@ -49,6 +49,8 @@ define([ $: $ }; + var CHECKPOINT_INTERVAL = 50; + var stringify = function (obj) { return JSONSortify(obj); }; @@ -58,11 +60,66 @@ define([ var andThen = function (common) { var sframeChan = common.getSframeChannel(); var metadataMgr = common.getMetadataMgr(); + var privateData = metadataMgr.getPrivateData(); var readOnly = false; - var locked = false; + //var locked = false; var config = {}; - var hashes = []; - var channel; + var content = { + hashes: {}, + ids: {} + }; + var myOOId; + + var deleteOffline = function () { + var ids = content.ids; + var users = Object.keys(metadataMgr.getMetadata().users); + Object.keys(ids).forEach(function (id) { + var nId = id.slice(0,32); + if (users.indexOf(nId) === -1) { + delete ids[id]; + } + }); + APP.onLocal(); + }; + + var isUserOnline = function (ooid) { + // Remove ids for users that have left the channel + deleteOffline(); + var ids = content.ids; + // Check if the provided id is in the ID list + return Object.keys(ids).some(function (id) { + return ooid === ids[id]; + }); + }; + + var setMyId = function (netfluxId) { + // Remove ids for users that have left the channel + deleteOffline(); + var ids = content.ids; + if (!myOOId) { + myOOId = Util.createRandomInteger(); + while (Object.keys(ids).some(function (id) { + return ids[id] === myOOId; + })) { + myOOId = Util.createRandomInteger(); + } + } + var myId = (netfluxId || metadataMgr.getNetfluxId()) + '-' + privateData.clientId; + ids[myId] = myOOId; + APP.onLocal(); + }; + + // Another tab from our worker has left: remove its id from the list + var removeClient = function (obj) { + var tabId = metadataMgr.getNetfluxId() + '-' + obj.id; + console.log(tabId); + if (content.ids[tabId]) { + console.log('delete'); + delete content.ids[tabId]; + APP.onLocal(); + console.log(content.ids); + } + }; var getFileType = function () { var type = common.getMetadataMgr().getPrivateData().ooType; @@ -91,12 +148,12 @@ define([ var now = function () { return +new Date(); }; var getLastCp = function () { - if (!hashes || !hashes.length) { return; } - var last = hashes.slice().pop(); - var parsed = Hash.parsePadUrl(last); - var secret = Hash.getSecrets('file', parsed.hash); - if (!secret || !secret.channel) { return; } - return 'cp|' + secret.channel.slice(0,8); + var hashes = content.hashes; + if (!hashes || !Object.keys(hashes).length) { return {}; } + var lastIndex = Math.max.apply(null, Object.keys(hashes).map(Number)); + // TODO check if hashes[lastIndex] is undefined? + var last = JSON.parse(JSON.stringify(hashes[lastIndex])); + return last; }; var rtChannel = { @@ -119,32 +176,112 @@ define([ var ooChannel = { ready: false, queue: [], - send: function () {} + send: function () {}, + cpIndex: 0 + }; + + var getContent = APP.getContent = function () { + try { + return window.frames[0].editor.asc_nativeGetFile(); + } catch (e) { + console.error(e); + return; + } + }; + + var fmConfig = { + noHandlers: true, + noStore: true, + body: $('body'), + onUploaded: function (ev, data) { + if (!data || !data.url) { return; } + sframeChan.query('Q_OO_SAVE', data, function (err) { + if (err) { + console.error(err); + return void UI.alert(Messages.oo_saveError); + } + var i = Math.floor(ev.index / CHECKPOINT_INTERVAL); + // XXX check if content.hashes[i] already exists? + content.hashes[i] = { + file: data.url, + hash: ev.hash, + index: ev.index + }; + content.saveLock = undefined; + APP.onLocal(); + sframeChan.query('Q_OO_COMMAND', { + cmd: 'UPDATE_HASH', + data: ev.hash + }, function (err, obj) { + if (err || (obj && obj.error)) { console.error(err || obj.error); } + }); + UI.log(Messages.saved); + }); + } + }; + APP.FM = common.createFileManager(fmConfig); + + var saveToServer = function () { + var text = getContent(); + var blob = new Blob([text], {type: 'plain/text'}); + var file = getFileType(); + blob.name = (metadataMgr.getMetadataLazy().title || file.doc) + '.' + file.type; + var data = { + hash: ooChannel.lastHash, + index: ooChannel.cpIndex + }; + APP.FM.handleFile(blob, data); + }; + var makeCheckpoint = function (force) { + var locked = content.saveLock; + if (!locked || !isUserOnline(locked) || force) { + content.saveLock = myOOId; + APP.onLocal(); + APP.realtime.onSettle(function () { + saveToServer(); + }); + return; + } + // The save is locked by someone else. If no new checkpoint is created + // in the next 20 to 40 secondes and the lock is kept by the same user, + // force the lock and make a checkpoint. + var saved = stringify(content.hashes); + var to = 20000 + (Math.random() * 20000) + setTimeout(function () { + if (stringify(content.hashes) === saved && locked === content.saveLock) { + makeCheckpoint(force); + } + }, to); }; var openRtChannel = function (cb) { - if (rtChannel.ready) { return; } - var chan = channel || Hash.createChannelId(); - if (!channel) { - channel = chan; + if (rtChannel.ready) { return void cb(); } + var chan = content.channel || Hash.createChannelId(); + if (!content.channel) { + content.channel = chan; APP.onLocal(); } sframeChan.query('Q_OO_OPENCHANNEL', { - channel: channel, - lastCp: getLastCp() + channel: content.channel, + lastCpHash: getLastCp().hash }, function (err, obj) { if (err || (obj && obj.error)) { console.error(err || (obj && obj.error)); } }); - sframeChan.on('EV_OO_EVENT', function (data) { - switch (data.ev) { + sframeChan.on('EV_OO_EVENT', function (obj) { + switch (obj.ev) { case 'READY': rtChannel.ready = true; break; + case 'LEAVE': + removeClient(obj.data); + break; case 'MESSAGE': if (ooChannel.ready) { - ooChannel.send(data.data); + ooChannel.send(obj.data.msg); + ooChannel.lastHash = obj.data.hash; + ooChannel.cpIndex++; } else { - ooChannel.queue.push(data.data); + ooChannel.queue.push(obj.data); } break; } @@ -168,7 +305,7 @@ define([ }; }); }; - var mkChannel = function () { + var makeChannel = function () { var msgEv = Util.mkEvent(); var iframe = $('#cp-app-oo-container > iframe')[0].contentWindow; window.addEventListener('message', function (msg) { @@ -213,8 +350,10 @@ define([ }); setTimeout(function () { if (ooChannel.queue) { - ooChannel.queue.forEach(function (msg) { - send(msg); + ooChannel.queue.forEach(function (data) { + send(data.msg); + ooChannel.lastHash = data.hash; + ooChannel.cpIndex++; }); } }, 2000); @@ -253,7 +392,12 @@ define([ changesIndex: 2, locks: [], // XXX take from userdoc? excelAdditionalInfo: null - }, null, function () { + }, null, function (err, hash) { + ooChannel.cpIndex++; + ooChannel.lastHash = hash; + if (ooChannel.cpIndex % CHECKPOINT_INTERVAL === 0) { + makeCheckpoint(); + } }); break; } @@ -261,11 +405,11 @@ define([ }); }; + var ooLoaded = false; var startOO = function (blob, file) { if (APP.ooconfig) { return void console.error('already started'); } var url = URL.createObjectURL(blob); - var lock = /*locked !== common.getMetadataMgr().getNetfluxId() ||*/ - !common.isLoggedIn(); + var lock = readOnly || !common.isLoggedIn(); // Config APP.ooconfig = { @@ -313,51 +457,15 @@ define([ if (ifr) { ifr.remove(); } }; APP.docEditor = new DocsAPI.DocEditor("cp-app-oo-placeholder", APP.ooconfig); - mkChannel(); - }; - - var getContent = APP.getContent = function () { - try { - return window.frames[0].editor.asc_nativeGetFile(); - } catch (e) { - console.error(e); - return; - } - }; - - var fmConfig = { - noHandlers: true, - noStore: true, - body: $('body'), - onUploaded: function (ev, data) { - if (!data || !data.url) { return; } - common.getSframeChannel().query('Q_OO_SAVE', data, function (err) { - if (err) { - console.error(err); - return void UI.alert(Messages.oo_saveError); - } - hashes.push(data.url); - APP.onLocal(); - rtChannel.sendMsg(null, getLastCp(), function () { - UI.log(Messages.saved); - }); - }); - } - }; - APP.FM = common.createFileManager(fmConfig); - - var saveToServer = function () { - var text = getContent(); - var blob = new Blob([text], {type: 'plain/text'}); - var file = getFileType(); - blob.name = (metadataMgr.getMetadataLazy().title || file.doc) + '.' + file.type; - APP.FM.handleFile(blob); + ooLoaded = true; + makeChannel(); }; var loadLastDocument = function () { - if (!hashes || !hashes.length) { return; } - var last = hashes.slice().pop(); - var parsed = Hash.parsePadUrl(last); + var lastCp = getLastCp(); + if (!lastCp) { return; } + ooChannel.cpIndex = lastCp.index || 0; + var parsed = Hash.parsePadUrl(lastCp.file); var secret = Hash.getSecrets('file', parsed.hash); if (!secret || !secret.channel) { return; } var hexFileName = secret.channel; @@ -383,6 +491,7 @@ define([ xhr.send(null); }; var loadDocument = function (newPad) { + if (ooLoaded) { return; } var type = common.getMetadataMgr().getPrivateData().ooType; var file = getFileType(); if (!newPad) { @@ -432,17 +541,15 @@ define([ var stringifyInner = function () { var obj = { - content: { - channel: channel, - hashes: hashes || [], - //locked: locked - }, + content: content, metadata: metadataMgr.getMetadataLazy() }; // stringify the json and send it into chainpad return stringify(obj); }; + APP.getContent = function () { return content; }; + APP.onLocal = config.onLocal = function () { if (initializing) { return; } if (readOnly) { return; } @@ -482,7 +589,9 @@ define([ var $rightside = toolbar.$rightside; - var $save = common.createButton('save', true, {}, saveToServer); + var $save = common.createButton('save', true, {}, function () { + saveToServer(); + }); $save.appendTo($rightside); if (common.isLoggedIn()) { @@ -520,10 +629,8 @@ define([ UI.errorLoadingScreen(errorText); throw new Error(errorText); } - hashes = hjson.content && hjson.content.hashes; - //locked = hjson.content && hjson.content.locked; - channel = hjson.content && hjson.content.channel; - newDoc = !hashes || hashes.length === 0; + content = hjson.content || content; + newDoc = !content.hashes || Object.keys(content.hashes).length === 0; } else { Title.updateTitle(Title.defaultTitle); } @@ -551,15 +658,14 @@ define([ openRtChannel(function () { loadDocument(newDoc); - initializing = false; + setMyId(); setEditable(!readOnly); UI.removeLoadingScreen(); }); }; - var reloadDisplayed = false; config.onRemote = function () { if (initializing) { return; } var userDoc = APP.realtime.getUserDoc(); @@ -567,20 +673,7 @@ define([ if (json.metadata) { metadataMgr.updateMetadata(json.metadata); } - /* - var newHashes = (json.content && json.content.hashes) || []; - if (newHashes.length !== hashes.length || - stringify(newHashes) !== stringify(hashes)) { - hashes = newHashes; - if (reloadDisplayed) { return; } - reloadDisplayed = true; - UI.confirm(Messages.oo_newVersion, function (yes) { - reloadDisplayed = false; - if (!yes) { return; } - common.gotoURL(); - }); - } - */ + content = json.content; }; config.onAbort = function () { diff --git a/www/common/onlyoffice/main.js b/www/common/onlyoffice/main.js index 3188da5a2..2da75c239 100644 --- a/www/common/onlyoffice/main.js +++ b/www/common/onlyoffice/main.js @@ -63,7 +63,7 @@ define([ // XXX add owners? // owners: something... channel: data.channel, - lastCp: data.lastCp, + lastCpHash: data.lastCpHash, padChan: Utils.secret.channel, validateKey: Utils.secret.keys.validateKey } @@ -71,18 +71,22 @@ define([ }); sframeChan.on('Q_OO_COMMAND', function (obj, cb) { if (obj.cmd === 'SEND_MESSAGE') { - if (obj.data.isCp) { - obj.data.isCp += '|' + crypto.encrypt('cp'); - } else { - obj.data.msg = crypto.encrypt(JSON.stringify(obj.data.msg)); - } + obj.data.msg = crypto.encrypt(JSON.stringify(obj.data.msg)); + var hash = obj.data.msg.slice(0,64); + var _cb = cb; + cb = function () { + _cb(hash); + }; } Cryptpad.onlyoffice.execCommand(obj, cb); }); Cryptpad.onlyoffice.onEvent.reg(function (obj) { - if (obj.ev === 'MESSAGE') { + if (obj.ev === 'MESSAGE' && !/^cp\|/.test(obj.data)) { try { - obj.data = JSON.parse(crypto.decrypt(obj.data, Utils.secret.keys.validateKey)); + obj.data = { + msg: JSON.parse(crypto.decrypt(obj.data, Utils.secret.keys.validateKey)), + hash: obj.data.slice(0,64) + }; } catch (e) { console.error(e); } diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index d1b63ae05..014971fb8 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -454,6 +454,7 @@ define([ }, // "priv" is not shared with other users but is needed by the apps priv: { + clientId: clientId, edPublic: store.proxy.edPublic, friends: store.proxy.friends || {}, settings: store.proxy.settings, diff --git a/www/common/outer/onlyoffice.js b/www/common/outer/onlyoffice.js index b465eff43..c08135b20 100644 --- a/www/common/outer/onlyoffice.js +++ b/www/common/outer/onlyoffice.js @@ -1,11 +1,7 @@ define([ - '/common/common-util.js', -], function (Util) { +], function () { var OO = {}; - var getHistory = function (ctx, data, clientId, cb) { - }; - var openChannel = function (ctx, obj, client, cb) { var channel = obj.channel; var padChan = obj.padChan; @@ -29,7 +25,6 @@ define([ // ==> Use our netflux ID to create our client ID if (!c.id) { c.id = chan.wc.myID + '-' + client; } - /// XXX send chan.history to client chan.history.forEach(function (msg) { ctx.emit('MESSAGE', msg, [client]); }); @@ -61,17 +56,11 @@ define([ } wc.on('join', function () { - // XXX }); - wc.on('leave', function (peer) { - // XXX + wc.on('leave', function () { }); wc.on('message', function (msg) { - if (/^cp\|/.test(msg)) { - chan.history = []; - } else { - chan.history.push(msg); - } + chan.history.push(msg); ctx.emit('MESSAGE', msg, chan.clients); }); @@ -79,11 +68,7 @@ define([ chan.sendMsg = function (msg, cb) { cb = cb || function () {}; wc.bcast(msg).then(function () { - if (/^cp\|/.test(msg)) { - chan.history = []; - } else { - chan.history.push(msg); - } + chan.history.push(msg); cb(); }, function (err) { cb({error: err}); @@ -92,7 +77,7 @@ define([ if (first) { chan.clients = [client]; - chan.lastCp = obj.lastCp; + chan.lastCpHash = obj.lastCpHash; first = false; cb(); } @@ -100,7 +85,7 @@ define([ var hk = network.historyKeeper; var cfg = { validateKey: obj.validateKey, - lastKnownHash: chan.lastKnownHash, + lastKnownHash: chan.lastKnownHash || chan.lastCpHash, owners: obj.owners, }; var msg = ['GET_HISTORY', wc.id, cfg]; @@ -137,24 +122,11 @@ define([ } if (parsed.error && parsed.channel) { return; } - var msg = parsed[4]; + msg = parsed[4]; // Keep only the history for our channel if (parsed[3] !== channel) { return; } - if (chan.lastCp) { - if (chan.lastCp === msg.slice(0, 11)) { - delete chan.lastCp; - } - return; - } - - var isCp = /^cp\|/.test(msg); - if (isCp) { - chan.history = []; - return; - } - chan.lastKnownHash = msg.slice(0,64); ctx.emit('MESSAGE', msg, chan.clients); chan.history.push(msg); @@ -172,6 +144,25 @@ define([ }); }; + var updateHash = function (ctx, data, clientId, cb) { + var c = ctx.clients[clientId]; + if (!c) { return void cb({ error: 'NOT_IN_CHANNEL' }); } + var chan = ctx.channels[c.channel]; + if (!chan) { return void cb({ error: 'INVALID_CHANNEL' }); } + var hash = data; + var index = -1; + chan.history.some(function (msg, idx) { + if (msg.slice(0,64) === hash) { + index = idx + 1; + return true; + } + }); + if (index !== -1) { + chan.history = chan.history.slice(index); + } + cb(); + }; + var sendMessage = function (ctx, data, clientId, cb) { var c = ctx.clients[clientId]; if (!c) { return void cb({ error: 'NOT_IN_CHANNEL' }); } @@ -214,6 +205,11 @@ define([ } } + var oldChannel = ctx.clients[clientId].channel; + var oldChan = ctx.channels[oldChannel]; + if (oldChan) { + ctx.emit('LEAVE', {id: clientId}, [oldChan.clients[0]]); + } delete ctx.clients[clientId]; }; @@ -240,6 +236,9 @@ define([ if (cmd === 'SEND_MESSAGE') { return void sendMessage(ctx, data, clientId, cb); } + if (cmd === 'UPDATE_HASH') { + return void updateHash(ctx, data, clientId, cb); + } if (cmd === 'OPEN_CHANNEL') { return void openChannel(ctx, data, clientId, cb); } diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 118fe5652..9512f1e61 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -95,7 +95,6 @@ define([ // sframe-boot.js. Then we can start the channel. var msgEv = _Util.mkEvent(); var iframe = $('#sbox-iframe')[0].contentWindow; - var iframeReady = false; var postMsg = function (data) { iframe.postMessage(data, '*'); };