From bd24821c6c42b9550bf3817a22b9c1ddeec5c0dd Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 22 Mar 2016 10:06:42 +0100 Subject: [PATCH 01/69] Don't attempt to use the cursor selection when it has length 0 RTWYSIWYG-20 RTWYSIWYG-24 --- www/common/cursor.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/www/common/cursor.js b/www/common/cursor.js index 583a8860b..069f896c7 100644 --- a/www/common/cursor.js +++ b/www/common/cursor.js @@ -136,10 +136,9 @@ define([ verbose("cursor.update"); root = root || inner; sel = sel || Rangy.getSelection(root); - // FIXME under what circumstances are no ranges found? if (!sel.rangeCount) { error('[cursor.update] no ranges found'); - //return 'no ranges found'; + return; } var range = sel.getRangeAt(0); From 79a9998b13de8917b339a390e43de75953d71765 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 22 Mar 2016 10:16:14 +0100 Subject: [PATCH 02/69] implement better serialization of class names RTWYSIWYG-27 : poorly formed yet valid HTML caused hyperjson to produce element selectors which hyperscript could not parse. --- www/common/hyperjson.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/www/common/hyperjson.js b/www/common/hyperjson.js index db6d3f20f..2f0badbd4 100644 --- a/www/common/hyperjson.js +++ b/www/common/hyperjson.js @@ -17,7 +17,7 @@ define([], function () { var callOnHyperJSON = function (hj, cb) { var children; - + if (hj && hj[2]) { children = hj[2].map(function (child) { if (isArray(child)) { @@ -39,6 +39,14 @@ define([], function () { return cb(hj[0], hj[1], children); }; + var prependDot = function (token) { + return '.' + token; + }; + + var isTruthy = function (x) { + return x; + }; + var DOM2HyperJSON = function(el){ if(!el.tagName && el.nodeType === Node.TEXT_NODE){ return el.textContent; @@ -73,7 +81,14 @@ define([], function () { delete attributes.id; } if(attributes.class){ - sel = sel +'.'+ attributes.class.replace(/ /g,"."); + /* TODO this can be done with RegExps alone, and it will be faster + but this works and is a little less error prone, albeit slower + come back and speed it up when it comes time to optimize */ + sel = sel +'.'+ attributes.class + .split(/\s+/) + .split(isTruthy) + .map(prependDot) + .join(''); delete attributes.class; } result.push(sel); From 0c6222b5f9d0de3b233fc6721b3eb6a304a3a392 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 22 Mar 2016 10:19:13 +0100 Subject: [PATCH 03/69] better error reporting when the operational transform fails to parse JSON --- www/common/json-ot.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/www/common/json-ot.js b/www/common/json-ot.js index 167e3d11a..238ccc18a 100644 --- a/www/common/json-ot.js +++ b/www/common/json-ot.js @@ -12,7 +12,12 @@ define([ JSON.parse(text3); return resultOp; } catch (e) { - console.log(e); + console.error(e); + console.log({ + resultOp: resultOp, + text2: text2, + text3: text3 + }); } // returning **null** breaks out of the loop From b2753ef7b74cd88f42b7d269cf4544418a27af1e Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 22 Mar 2016 19:28:50 +0100 Subject: [PATCH 04/69] fix string manipulation off-by-one --- www/common/hyperjson.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/common/hyperjson.js b/www/common/hyperjson.js index 2f0badbd4..35975aede 100644 --- a/www/common/hyperjson.js +++ b/www/common/hyperjson.js @@ -84,7 +84,7 @@ define([], function () { /* TODO this can be done with RegExps alone, and it will be faster but this works and is a little less error prone, albeit slower come back and speed it up when it comes time to optimize */ - sel = sel +'.'+ attributes.class + sel = sel + attributes.class .split(/\s+/) .split(isTruthy) .map(prependDot) From 1bd5cb9e27c854b1a4a883f1773f67545258acba Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 23 Mar 2016 12:31:16 +0100 Subject: [PATCH 05/69] hyperjson.js : used split instead of filter... oops --- www/common/hyperjson.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/www/common/hyperjson.js b/www/common/hyperjson.js index 35975aede..660c62194 100644 --- a/www/common/hyperjson.js +++ b/www/common/hyperjson.js @@ -84,9 +84,10 @@ define([], function () { /* TODO this can be done with RegExps alone, and it will be faster but this works and is a little less error prone, albeit slower come back and speed it up when it comes time to optimize */ + sel = sel + attributes.class .split(/\s+/) - .split(isTruthy) + .filter(isTruthy) .map(prependDot) .join(''); delete attributes.class; From 0d33af773f43902faef0d7621f7f2254931ae953 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 24 Mar 2016 12:11:31 +0100 Subject: [PATCH 06/69] implement optional filtering in hyperjson Implemented via callback, return falsey if you want to filter an element and all of its children from the serialized result. --- www/common/convert.js | 4 ++-- www/common/hyperjson.js | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/www/common/convert.js b/www/common/convert.js index b95591e93..973881c80 100644 --- a/www/common/convert.js +++ b/www/common/convert.js @@ -5,7 +5,7 @@ define([ ], function (vdom, hyperjson, hyperscript) { // complain if you don't find the required APIs if (!(vdom && hyperjson && hyperscript)) { throw new Error(); } - + // Generate a matrix of conversions /* convert.dom.to.hjson, convert.hjson.to.dom, @@ -46,7 +46,7 @@ define([ return hyperjson.fromDOM(vdom.create(V)); } } - }, + }, convert = {}; Object.keys(methods).forEach(function (method) { convert[method] = { to: methods[method] }; diff --git a/www/common/hyperjson.js b/www/common/hyperjson.js index 660c62194..31a2caf08 100644 --- a/www/common/hyperjson.js +++ b/www/common/hyperjson.js @@ -47,13 +47,20 @@ define([], function () { return x; }; - var DOM2HyperJSON = function(el){ + var DOM2HyperJSON = function(el, predicate){ if(!el.tagName && el.nodeType === Node.TEXT_NODE){ return el.textContent; } if(!el.attributes){ return; } + + if (predicate) { + if (!predicate(el)) { + // shortcircuit + return; + } + } var attributes = {}; var i = 0; @@ -102,10 +109,12 @@ define([], function () { // js hint complains if we use 'var' here i = 0; + for(; i < el.childNodes.length; i++){ - children.push(DOM2HyperJSON(el.childNodes[i])); + children.push(DOM2HyperJSON(el.childNodes[i], predicate)); } - result.push(children); + + result.push(children.filter(isTruthy)); return result; }; From 4b35a145e31bf375d7889a309e3e97e416ac9656 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 25 Mar 2016 11:04:27 +0100 Subject: [PATCH 07/69] Push WIP --- www/_socket/index.html | 41 +++ www/_socket/inner.html | 12 + www/_socket/main.js | 311 ++++++++++++++++ www/_socket/realtime-input.js | 346 ++++++++++++++++++ .../sharejs_textarea-transport-only.js | 74 ++++ www/_socket/toolbar.js | 233 ++++++++++++ 6 files changed, 1017 insertions(+) create mode 100644 www/_socket/index.html create mode 100644 www/_socket/inner.html create mode 100644 www/_socket/main.js create mode 100644 www/_socket/realtime-input.js create mode 100644 www/_socket/sharejs_textarea-transport-only.js create mode 100644 www/_socket/toolbar.js diff --git a/www/_socket/index.html b/www/_socket/index.html new file mode 100644 index 000000000..6ee2a8596 --- /dev/null +++ b/www/_socket/index.html @@ -0,0 +1,41 @@ + + + + + + + + + + + + + diff --git a/www/_socket/inner.html b/www/_socket/inner.html new file mode 100644 index 000000000..bf79dcd0d --- /dev/null +++ b/www/_socket/inner.html @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/www/_socket/main.js b/www/_socket/main.js new file mode 100644 index 000000000..56f03cc1e --- /dev/null +++ b/www/_socket/main.js @@ -0,0 +1,311 @@ +define([ + '/api/config?cb=' + Math.random().toString(16).substring(2), + '/common/messages.js', + '/common/crypto.js', + '/_socket/realtime-input.js', + '/common/convert.js', + '/_socket/toolbar.js', + '/common/cursor.js', + '/common/json-ot.js', + '/bower_components/diff-dom/diffDOM.js', + '/bower_components/jquery/dist/jquery.min.js', + '/customize/pad.js' +], function (Config, Messages, Crypto, realtimeInput, Convert, Toolbar, Cursor, JsonOT) { + var $ = window.jQuery; + var ifrw = $('#pad-iframe')[0].contentWindow; + var Ckeditor; // to be initialized later... + var DiffDom = window.diffDOM; + + window.Convert = Convert; + + window.Toolbar = Toolbar; + + var userName = Crypto.rand64(8), + toolbar; + + var module = {}; + + var isNotMagicLine = function (el) { + // factor as: + // return !(el.tagName === 'SPAN' && el.contentEditable === 'false'); + var filter = (el.tagName === 'SPAN' && el.contentEditable === 'false'); + if (filter) { + console.log("[hyperjson.serializer] prevented an element" + + "from being serialized:", el); + return false; + } + return true; + }; + + var andThen = function (Ckeditor) { + $(window).on('hashchange', function() { + window.location.reload(); + }); + if (window.location.href.indexOf('#') === -1) { + window.location.href = window.location.href + '#' + Crypto.genKey(); + return; + } + + var fixThings = false; + var key = Crypto.parseKey(window.location.hash.substring(1)); + var editor = window.editor = Ckeditor.replace('editor1', { + // https://dev.ckeditor.com/ticket/10907 + needsBrFiller: fixThings, + needsNbspFiller: fixThings, + removeButtons: 'Source,Maximize', + // magicline plugin inserts html crap into the document which is not part of the + // document itself and causes problems when it's sent across the wire and reflected back + // but we filter it now, so that's ok. + removePlugins: 'resize' + }); + + editor.on('instanceReady', function (Ckeditor) { + editor.execCommand('maximize'); + var documentBody = ifrw.$('iframe')[0].contentDocument.body; + + documentBody.innerHTML = Messages.initialState; + + var inner = window.inner = documentBody; + var cursor = window.cursor = Cursor(inner); + + var $textarea = $('#feedback'); + + var setEditable = function (bool) { + // inner.style.backgroundColor = bool? 'unset': 'grey'; + inner.setAttribute('contenteditable', bool); + }; + + // don't let the user edit until the pad is ready + setEditable(false); + + var diffOptions = { + preDiffApply: function (info) { + /* TODO DiffDOM will filter out magicline plugin elements + in practice this will make it impossible to use it + while someone else is typing, which could be annoying + + we should check when such an element is going to be + removed, and prevent that from happening. */ + + // no use trying to recover the cursor if it doesn't exist + if (!cursor.exists()) { return; } + + /* frame is either 0, 1, 2, or 3, depending on which + cursor frames were affected: none, first, last, or both + */ + var frame = info.frame = cursor.inNode(info.node); + + if (!frame) { return; } + + if (typeof info.diff.oldValue === 'string' && typeof info.diff.newValue === 'string') { + var pushes = cursor.pushDelta(info.diff.oldValue, info.diff.newValue); + + if (frame & 1) { + // push cursor start if necessary + if (pushes.commonStart < cursor.Range.start.offset) { + cursor.Range.start.offset += pushes.delta; + } + } + if (frame & 2) { + // push cursor end if necessary + if (pushes.commonStart < cursor.Range.end.offset) { + cursor.Range.end.offset += pushes.delta; + } + } + } + }, + postDiffApply: function (info) { + if (info.frame) { + if (info.node) { + if (info.frame & 1) { cursor.fixStart(info.node); } + if (info.frame & 2) { cursor.fixEnd(info.node); } + } else { console.error("info.node did not exist"); } + + var sel = cursor.makeSelection(); + var range = cursor.makeRange(); + + cursor.fixSelection(sel, range); + } + } + }; + + var initializing = true; + + var assertStateMatches = function () { + var userDocState = module.realtimeInput.realtime.getUserDoc(); + var currentState = $textarea.val(); + if (currentState !== userDocState) { + console.log({ + userDocState: userDocState, + currentState: currentState + }); + throw new Error("currentState !== userDocState"); + } + }; + + var updateDebugTextarea = function (shjson) { + window.setTimeout(function () { + $textarea.val(shjson); + }, 0); + }; + + // apply patches, and try not to lose the cursor in the process! + var applyHjson = function (shjson) { + setEditable(false); + var userDocStateDom = Convert.hjson.to.dom(JSON.parse(shjson)); + userDocStateDom.setAttribute("contenteditable", "true"); // lol wtf + var DD = new DiffDom(diffOptions); + + //assertStateMatches(); + + var patch = (DD).diff(inner, userDocStateDom); + (DD).apply(inner, patch); + + // push back to the textarea so we get a userDocState + setEditable(true); + }; + + var onRemote = function (info) { + if (initializing) { return; } + + var shjson = info.realtime.getUserDoc(); + + // remember where the cursor is + cursor.update(); + + // build a dom from HJSON, diff, and patch the editor + applyHjson(shjson); + //updateDebugTextarea(shjson); + }; + + var onInit = function (info) { + var $bar = $('#pad-iframe')[0].contentWindow.$('#cke_1_toolbox'); + toolbar = info.realtime.toolbar = Toolbar.create($bar, userName, info.realtime); + /* TODO handle disconnects and such*/ + }; + + var onReady = function (info) { + console.log("Unlocking editor"); + initializing = false; + setEditable(true); + + var shjson = info.realtime.getUserDoc(); + + applyHjson(shjson); + }; + + var onAbort = function (info) { + console.log("Aborting the session!"); + // stop the user from continuing to edit + setEditable(false); + // TODO inform them that the session was torn down + toolbar.failed(); + }; + + var realtimeOptions = { + // configuration :D + doc: inner, + // first thing called + onInit: onInit, + + onReady: onReady, + + // when remote changes occur + onRemote: onRemote, + + // handle aborts + onAbort: onAbort, + + // provide initialstate... + initialState: JSON.stringify(Convert.core.hyperjson.fromDOM(inner, isNotMagicLine)), + + // really basic operational transform + // reject patch if it results in invalid JSON + transformFunction : JsonOT.validate, + + // websocketURL, ofc + websocketURL: Config.websocketURL, + + // username + userName: userName, + + // communication channel name + channel: key.channel, + + // encryption key + cryptKey: key.cryptKey + }; + + var rti = module.realtimeInput = window.rti = realtimeInput.start(realtimeOptions); + + var propogate = function () { + var hjson = Convert.core.hyperjson.fromDOM(inner, isNotMagicLine); + var shjson = JSON.stringify(hjson); + + rti.propogate(shjson); + rti.onEvent(shjson); + }; + + var testInput = window.testInput = function (el, offset) { + var i = 0, + j = offset, + input = "The quick red fox jumped over the lazy brown dog. ", + l = input.length, + errors = 0, + max_errors = 15, + interval; + var cancel = function () { + if (interval) { window.clearInterval(interval); } + }; + + interval = window.setInterval(function () { + propogate(); + try { + el.replaceData(j, 0, input.charAt(i)); + } catch (err) { + errors++; + if (errors >= max_errors) { + console.log("Max error number exceeded"); + cancel(); + } + + console.error(err); + var next = document.createTextNode(""); + el.parentNode.appendChild(next); + el = next; + j = 0; + } + i = (i + 1) % l; + j++; + }, 200); + + return { + cancel: cancel + }; + }; + + var easyTest = window.easyTest = function () { + cursor.update(); + var start = cursor.Range.start; + var test = testInput(start.el, start.offset); + propogate(); + return test; + }; + + editor.on('change', propogate); + }); + }; + + var interval = 100; + var first = function () { + Ckeditor = ifrw.CKEDITOR; + if (Ckeditor) { + andThen(Ckeditor); + } else { + console.log("Ckeditor was not defined. Trying again in %sms",interval); + setTimeout(first, interval); + } + }; + + $(first); +}); diff --git a/www/_socket/realtime-input.js b/www/_socket/realtime-input.js new file mode 100644 index 000000000..4cc66227b --- /dev/null +++ b/www/_socket/realtime-input.js @@ -0,0 +1,346 @@ +/* + * Copyright 2014 XWiki SAS + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +define([ + '/common/messages.js', + '/bower_components/reconnectingWebsocket/reconnecting-websocket.js', + '/common/crypto.js', + '/_socket/toolbar.js', + '/_socket/sharejs_textarea-transport-only.js', + '/common/chainpad.js', + '/bower_components/jquery/dist/jquery.min.js', +], function (Messages,/*FIXME*/ ReconnectingWebSocket, Crypto, Toolbar, sharejs) { + var $ = window.jQuery; + var ChainPad = window.ChainPad; + var PARANOIA = true; + var module = { exports: {} }; + + /** + * If an error is encountered but it is recoverable, do not immediately fail + * but if it keeps firing errors over and over, do fail. + */ + var MAX_RECOVERABLE_ERRORS = 15; + + var recoverableErrors = 0; + + /** Maximum number of milliseconds of lag before we fail the connection. */ + var MAX_LAG_BEFORE_DISCONNECT = 20000; + + var debug = function (x) { console.log(x); }; + var warn = function (x) { console.error(x); }; + var verbose = function (x) { /*console.log(x);*/ }; + var error = function (x) { + console.error(x); + recoverableErrors++; + if (recoverableErrors >= MAX_RECOVERABLE_ERRORS) { + alert("FAIL"); + } + }; + + // ------------------ Trapping Keyboard Events ---------------------- // + + var bindEvents = function (element, events, callback, unbind) { + for (var i = 0; i < events.length; i++) { + var e = events[i]; + if (element.addEventListener) { + if (unbind) { + element.removeEventListener(e, callback, false); + } else { + element.addEventListener(e, callback, false); + } + } else { + if (unbind) { + element.detachEvent('on' + e, callback); + } else { + element.attachEvent('on' + e, callback); + } + } + } + }; + + var bindAllEvents = function (textarea, docBody, onEvent, unbind) + { + /* + we use docBody for the purposes of CKEditor. + because otherwise special keybindings like ctrl-b and ctrl-i + would open bookmarks and info instead of applying bold/italic styles + */ + if (docBody) { + bindEvents(docBody, + ['textInput', 'keydown', 'keyup', 'select', 'cut', 'paste'], + onEvent, + unbind); + } + if (textarea) { + bindEvents(textarea, + ['mousedown','mouseup','click','change'], + onEvent, + unbind); + } + }; + + /* websocket stuff */ + var isSocketDisconnected = function (socket, realtime) { + var sock = socket._socket; + return sock.readyState === sock.CLOSING + || sock.readyState === sock.CLOSED + || (realtime.getLag().waiting && realtime.getLag().lag > MAX_LAG_BEFORE_DISCONNECT); + }; + + // this differs from other functions with similar names in that + // you are expected to pass a socket into it. + var checkSocket = function (socket) { + if (isSocketDisconnected(socket, socket.realtime) && + !socket.intentionallyClosing) { + return true; + } else { + return false; + } + }; + + // TODO before removing websocket implementation + // bind abort to onLeaving + var abort = function (socket, realtime) { + realtime.abort(); + try { socket._socket.close(); } catch (e) { warn(e); } + }; + + var handleError = function (socket, realtime, err, docHTML, allMessages) { + // var internalError = createDebugInfo(err, realtime, docHTML, allMessages); + abort(socket, realtime); + }; + + var makeWebsocket = function (url) { + var socket = new ReconnectingWebSocket(url); + /* create a set of handlers to use instead of the native socket handler + these handlers will iterate over all of the functions pushed to the + arrays bearing their name. + + The first such function to return `false` will prevent subsequent + functions from being executed. */ + var out = { + onOpen: [], // takes care of launching the post-open logic + onClose: [], // takes care of cleanup + onError: [], // in case of error, socket will close, and fire this + onMessage: [], // used for the bulk of our logic + send: function (msg) { socket.send(msg); }, + close: function () { socket.close(); }, + _socket: socket + }; + var mkHandler = function (name) { + return function (evt) { + for (var i = 0; i < out[name].length; i++) { + if (out[name][i](evt) === false) { + console.log(name +"Handler"); + return; + } + } + }; + }; + + // bind your new handlers to the important listeners on the socket + socket.onopen = mkHandler('onOpen'); + socket.onclose = mkHandler('onClose'); + socket.onerror = mkHandler('onError'); + socket.onmessage = mkHandler('onMessage'); + return out; + }; + /* end websocket stuff */ + + var start = module.exports.start = function (config) { + //var textarea = config.textarea; + var websocketUrl = config.websocketURL; + var userName = config.userName; + var channel = config.channel; + var cryptKey = config.cryptKey; + var passwd = 'y'; + var doc = config.doc || null; + + // wrap up the reconnecting websocket with our additional stack logic + var socket = makeWebsocket(websocketUrl); + + var allMessages = []; + var isErrorState = false; + var initializing = true; + var recoverableErrorCount = 0; + + var toReturn = { socket: socket }; + + socket.onOpen.push(function (evt) { + if (!initializing) { + console.log("Starting"); + // realtime is passed around as an attribute of the socket + // FIXME?? + socket.realtime.start(); + return; + } + + var realtime = toReturn.realtime = socket.realtime = + // everybody has a username, and we assume they don't collide + // usernames are used to determine whether a message is remote + // or local in origin. This could mess with expected behaviour + // if someone spoofed. + ChainPad.create(userName, + passwd, // password, to be deprecated (maybe) + channel, // the channel we're to connect to + + // initialState argument. (optional) + config.initialState || '', + + // transform function (optional), which handles conflicts + { transformFunction: config.transformFunction }); + + var onEvent = toReturn.onEvent = function (newText) { + if (isErrorState || initializing) { return; } + // assert things here... + if (realtime.getUserDoc() !== newText) { + // this is a problem +// warn("realtime.getUserDoc() !== newText"); + } + }; + + // pass your shiny new realtime into initialization functions + if (config.onInit) { + // extend as you wish + config.onInit({ + realtime: realtime + }); + } + + /* UI hints on userList changes are handled within the toolbar + so we don't actually need to do anything here except confirm + whether we've successfully joined the session, and call our + 'onReady' function */ + realtime.onUserListChange(function (userList) { + if (!initializing || userList.indexOf(userName) === -1) { + return; + } + // if we spot ourselves being added to the document, we'll switch + // 'initializing' off because it means we're fully synced. + initializing = false; + + // execute an onReady callback if one was supplied + // pass an object so we can extend this later + if (config.onReady) { + // extend as you wish + config.onReady({ + userList: userList, + realtime: realtime + }); + } + }); + + // when a message is ready to send + // Don't confuse this onMessage with socket.onMessage + realtime.onMessage(function (message) { + if (isErrorState) { return; } + message = Crypto.encrypt(message, cryptKey); + try { + socket.send(message); + } catch (e) { + warn(e); + } + }); + + // TODO improve this RegExp such that it allows for more names + // right now it only handles names generated by rand64() + var whoami = new RegExp(userName.replace(/[\/\+]/g, function (c) { + return '\\' +c; + })); + + // when you receive a message... + socket.onMessage.push(function (evt) { + verbose(evt.data); + if (isErrorState) { return; } + + var message = Crypto.decrypt(evt.data, cryptKey); + verbose(message); + allMessages.push(message); + if (!initializing) { + if (PARANOIA) { + // FIXME this is out of sync with the application logic + onEvent(); + } + } + realtime.message(message); + if (/\[5,/.test(message)) { verbose("pong"); } + + if (!initializing) { + if (/\[2,/.test(message)) { + //verbose("Got a patch"); + if (whoami.test(message)) { + //verbose("Received own message"); + } else { + //verbose("Received remote message"); + // obviously this is only going to get called if... XXX wat + if (config.onRemote) { config.onRemote({ + realtime: realtime + //realtime.getUserDoc() + }); } + } + } + } + }); + + // actual socket bindings + socket.onmessage = function (evt) { + for (var i = 0; i < socket.onMessage.length; i++) { + if (socket.onMessage[i](evt) === false) { return; } + } + }; + socket.onclose = function (evt) { + for (var i = 0; i < socket.onMessage.length; i++) { + if (socket.onClose[i](evt) === false) { return; } + } + }; + + socket.onerror = warn; + + var socketChecker = setInterval(function () { + if (checkSocket(socket)) { + warn("Socket disconnected!"); + + recoverableErrorCount += 1; + + if (recoverableErrorCount >= MAX_RECOVERABLE_ERRORS) { + warn("Giving up!"); + abort(socket, realtime); + if (config.onAbort) { + config.onAbort({ + socket: socket + }); + } + if (socketChecker) { clearInterval(socketChecker); } + } + } // it's working as expected, continue + }, 200); + + // TODO maybe push this out to the application layer. + bindAllEvents(null, doc, onEvent, false); + + // TODO rename 'sharejs.attach' to imply what we want to do + var genOp = toReturn.propogate = sharejs.attach({ + realtime: realtime + }); + + realtime.start(); + debug('started'); + }); + + return toReturn; + }; + return module.exports; +}); diff --git a/www/_socket/sharejs_textarea-transport-only.js b/www/_socket/sharejs_textarea-transport-only.js new file mode 100644 index 000000000..64d220051 --- /dev/null +++ b/www/_socket/sharejs_textarea-transport-only.js @@ -0,0 +1,74 @@ +define(function () { + +/* applyChange takes: + ctx: the context (aka the realtime) + oldval: the old value + newval: the new value + + it performs a diff on the two values, and generates patches + which are then passed into `ctx.remove` and `ctx.insert` + + + +*/ +var applyChange = function(ctx, oldval, newval) { + // Strings are immutable and have reference equality. I think this test is O(1), so its worth doing. + if (oldval === newval) { + return; + } + + var commonStart = 0; + while (oldval.charAt(commonStart) === newval.charAt(commonStart)) { + commonStart++; + } + + var commonEnd = 0; + while (oldval.charAt(oldval.length - 1 - commonEnd) === newval.charAt(newval.length - 1 - commonEnd) && + commonEnd + commonStart < oldval.length && commonEnd + commonStart < newval.length) { + commonEnd++; + } + + if (oldval.length !== commonStart + commonEnd) { + if (ctx.localChange) { ctx.localChange(true); } + ctx.remove(commonStart, oldval.length - commonStart - commonEnd); + } + if (newval.length !== commonStart + commonEnd) { + if (ctx.localChange) { ctx.localChange(true); } + ctx.insert(commonStart, newval.slice(commonStart, newval.length - commonEnd)); + } +}; + +var attachTextarea = function(config) { + var ctx = config.realtime; + + // initial state will always fail the !== check in genop. + // because nothing will equal this object + var content = {}; + + // FIXME this is only necessary because we need to be able to update the + // textarea. This is being deprecated, however. Instead + var replaceText = function(newText) { + content = newText; + }; + + // *** remote -> local changes + ctx.onRemove(function(pos, length) { + replaceText(ctx.getUserDoc()); + }); + + ctx.onInsert(function(pos, text) { + replaceText(ctx.getUserDoc()); + }); + + return function (newContent) { + if (newContent !== content) { + applyChange(ctx, ctx.getUserDoc(), newContent); + if (ctx.getUserDoc() !== newContent) { + console.log("Expected that: `ctx.getUserDoc() === newContent`!"); + } + } + }; +}; + +return { attach: attachTextarea }; +}); diff --git a/www/_socket/toolbar.js b/www/_socket/toolbar.js new file mode 100644 index 000000000..4159ea0ac --- /dev/null +++ b/www/_socket/toolbar.js @@ -0,0 +1,233 @@ +define([ + '/common/messages.js' +], function (Messages) { + + /** Id of the element for getting debug info. */ + var DEBUG_LINK_CLS = 'rtwysiwyg-debug-link'; + + /** Id of the div containing the user list. */ + var USER_LIST_CLS = 'rtwysiwyg-user-list'; + + /** Id of the div containing the lag info. */ + var LAG_ELEM_CLS = 'rtwysiwyg-lag'; + + /** The toolbar class which contains the user list, debug link and lag. */ + var TOOLBAR_CLS = 'rtwysiwyg-toolbar'; + + /** Key in the localStore which indicates realtime activity should be disallowed. */ + var LOCALSTORAGE_DISALLOW = 'rtwysiwyg-disallow'; + + var SPINNER_DISAPPEAR_TIME = 3000; + var SPINNER = [ '-', '\\', '|', '/' ]; + + var uid = function () { + return 'rtwysiwyg-uid-' + String(Math.random()).substring(2); + }; + + var createRealtimeToolbar = function ($container) { + var id = uid(); + $container.prepend( + '
' + + '
' + + '
' + + '
' + ); + var toolbar = $container.find('#'+id); + toolbar.append([ + '' + ].join('\n')); + return toolbar; + }; + + var createEscape = function ($container) { + var id = uid(); + $container.append('
⇐ Back
'); + var $ret = $container.find('#'+id); + $ret.on('click', function () { + window.location.href = '/'; + }); + return $ret[0]; + }; + + var createSpinner = function ($container) { + var id = uid(); + $container.append(''); + return $container.find('#'+id)[0]; + }; + + var kickSpinner = function (spinnerElement, reversed) { + var txt = spinnerElement.textContent || '-'; + var inc = (reversed) ? -1 : 1; + spinnerElement.textContent = SPINNER[(SPINNER.indexOf(txt) + inc) % SPINNER.length]; + if (spinnerElement.timeout) { clearTimeout(spinnerElement.timeout); } + spinnerElement.timeout = setTimeout(function () { + spinnerElement.textContent = ''; + }, SPINNER_DISAPPEAR_TIME); + }; + + var createUserList = function ($container) { + var id = uid(); + $container.append('
'); + return $container.find('#'+id)[0]; + }; + + var updateUserList = function (myUserName, listElement, userList) { + var meIdx = userList.indexOf(myUserName); + if (meIdx === -1) { + listElement.textContent = Messages.synchronizing; + return; + } + if (userList.length === 1) { + listElement.textContent = Messages.editingAlone; + } else if (userList.length === 2) { + listElement.textContent = Messages.editingWithOneOtherPerson; + } else { + listElement.textContent = Messages.editingWith + ' ' + (userList.length - 1) + ' ' + Messages.otherPeople; + } + }; + + var createLagElement = function ($container) { + var id = uid(); + $container.append('
'); + return $container.find('#'+id)[0]; + }; + + var checkLag = function (realtime, lagElement) { + var lag = realtime.getLag(); + var lagSec = lag.lag/1000; + var lagMsg = Messages.lag + ' '; + if (lag.waiting && lagSec > 1) { + lagMsg += "?? " + Math.floor(lagSec); + } else { + lagMsg += lagSec; + } + lagElement.textContent = lagMsg; + }; + + // this is a little hack, it should go in it's own file. + // FIXME ok, so let's put it in its own file then + // TODO there should also be a 'clear recent pads' button + var rememberPad = function () { + // FIXME, this is overly complicated, use array methods + var recentPadsStr = localStorage['CryptPad_RECENTPADS']; + var recentPads = []; + if (recentPadsStr) { recentPads = JSON.parse(recentPadsStr); } + // TODO use window.location.hash or something like that + if (window.location.href.indexOf('#') === -1) { return; } + var now = new Date(); + var out = []; + for (var i = recentPads.length; i >= 0; i--) { + if (recentPads[i] && + // TODO precompute this time value, maybe make it configurable? + // FIXME precompute the date too, why getTime every time? + now.getTime() - recentPads[i][1] < (1000*60*60*24*30) && + recentPads[i][0] !== window.location.href) + { + out.push(recentPads[i]); + } + } + out.push([window.location.href, now.getTime()]); + localStorage['CryptPad_RECENTPADS'] = JSON.stringify(out); + }; + + var create = function ($container, myUserName, realtime) { + var toolbar = createRealtimeToolbar($container); + createEscape(toolbar.find('.rtwysiwyg-toolbar-leftside')); + var userListElement = createUserList(toolbar.find('.rtwysiwyg-toolbar-leftside')); + var spinner = createSpinner(toolbar.find('.rtwysiwyg-toolbar-rightside')); + var lagElement = createLagElement(toolbar.find('.rtwysiwyg-toolbar-rightside')); + + rememberPad(); + + var connected = false; + + realtime.onUserListChange(function (userList) { + if (userList.indexOf(myUserName) !== -1) { connected = true; } + if (!connected) { return; } + updateUserList(myUserName, userListElement, userList); + }); + + var ks = function () { + if (connected) { kickSpinner(spinner, false); } + }; + + realtime.onPatch(ks); + // Try to filter out non-patch messages, doesn't have to be perfect this is just the spinner + realtime.onMessage(function (msg) { if (msg.indexOf(':[2,') > -1) { ks(); } }); + + setInterval(function () { + if (!connected) { return; } + checkLag(realtime, lagElement); + }, 3000); + + return { + failed: function () { + connected = false; + userListElement.textContent = ''; + lagElement.textContent = ''; + }, + reconnecting: function () { + connected = false; + userListElement.textContent = Messages.reconnecting; + lagElement.textContent = ''; + }, + connected: function () { + connected = true; + } + }; + }; + + return { create: create }; +}); From 420a7098a625c11fe32fc09191f6bb2c4c03e173 Mon Sep 17 00:00:00 2001 From: Caleb James DeLisle Date: Fri, 25 Mar 2016 12:45:51 +0100 Subject: [PATCH 08/69] more testing and crap --- www/_socket/main.js | 49 +++++++++++++------ www/_socket/realtime-input.js | 45 +++++++++-------- .../sharejs_textarea-transport-only.js | 13 ++--- www/common/chainpad.js | 1 + www/pad/realtime-wysiwyg.js | 4 +- 5 files changed, 68 insertions(+), 44 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index 56f03cc1e..d41274a98 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -37,6 +37,25 @@ define([ return true; }; + var setRandomizedInterval = function (func, target, range) { + var timeout; + var again = function () { + setTimeout(function () { + again(); + func(); + }, target - (range / 2) + Math.random() * range); + }; + again(); + return { + cancel: function () { + if (timeout) { + clearTimeout(timeout); + timeout = undefined; + } + } + }; + } + var andThen = function (Ckeditor) { $(window).on('hashchange', function() { window.location.reload(); @@ -149,20 +168,22 @@ define([ }, 0); }; + var now = function () { return new Date().getTime() }; + + var DD = new DiffDom(diffOptions); // apply patches, and try not to lose the cursor in the process! var applyHjson = function (shjson) { - setEditable(false); + //setEditable(false); + console.log(now()); var userDocStateDom = Convert.hjson.to.dom(JSON.parse(shjson)); userDocStateDom.setAttribute("contenteditable", "true"); // lol wtf - var DD = new DiffDom(diffOptions); - + console.log(now()); //assertStateMatches(); - var patch = (DD).diff(inner, userDocStateDom); (DD).apply(inner, patch); - + console.log(now()); // push back to the textarea so we get a userDocState - setEditable(true); + //setEditable(true); }; var onRemote = function (info) { @@ -238,11 +259,11 @@ define([ var rti = module.realtimeInput = window.rti = realtimeInput.start(realtimeOptions); - var propogate = function () { + var propogate = window.cryptpad_propogate = function () { var hjson = Convert.core.hyperjson.fromDOM(inner, isNotMagicLine); var shjson = JSON.stringify(hjson); - rti.propogate(shjson); + if (!rti.propogate(shjson)) { return; } rti.onEvent(shjson); }; @@ -255,10 +276,10 @@ define([ max_errors = 15, interval; var cancel = function () { - if (interval) { window.clearInterval(interval); } + //if (interval) { interval.cancel(); } }; - interval = window.setInterval(function () { + interval = setRandomizedInterval(function () { propogate(); try { el.replaceData(j, 0, input.charAt(i)); @@ -270,14 +291,14 @@ define([ } console.error(err); - var next = document.createTextNode(""); - el.parentNode.appendChild(next); + var next = document.createTextNode("-"); + window.inner.appendChild(next); el = next; - j = 0; + j = -1; } i = (i + 1) % l; j++; - }, 200); + }, 200, 50); return { cancel: cancel diff --git a/www/_socket/realtime-input.js b/www/_socket/realtime-input.js index 4cc66227b..3d36217b1 100644 --- a/www/_socket/realtime-input.js +++ b/www/_socket/realtime-input.js @@ -52,7 +52,7 @@ define([ // ------------------ Trapping Keyboard Events ---------------------- // - var bindEvents = function (element, events, callback, unbind) { + var _unused_bindEvents = function (element, events, callback, unbind) { for (var i = 0; i < events.length; i++) { var e = events[i]; if (element.addEventListener) { @@ -71,7 +71,7 @@ define([ } }; - var bindAllEvents = function (textarea, docBody, onEvent, unbind) + var _unused_bindAllEvents = function (textarea, docBody, onEvent, unbind) { /* we use docBody for the purposes of CKEditor. @@ -208,8 +208,10 @@ define([ // assert things here... if (realtime.getUserDoc() !== newText) { // this is a problem -// warn("realtime.getUserDoc() !== newText"); + warn("realtime.getUserDoc() !== newText"); } + //try{throw new Error();}catch(e){console.log(e.stack);} + console.log("2: " + realtime.Sha.hex_sha256(realtime.getUserDoc())); }; // pass your shiny new realtime into initialization functions @@ -255,11 +257,14 @@ define([ } }); - // TODO improve this RegExp such that it allows for more names - // right now it only handles names generated by rand64() - var whoami = new RegExp(userName.replace(/[\/\+]/g, function (c) { - return '\\' +c; - })); + realtime.onPatch(function () { + if (config.onRemote) { + config.onRemote({ + realtime: realtime + //realtime.getUserDoc() + }); + } + }); // when you receive a message... socket.onMessage.push(function (evt) { @@ -270,10 +275,11 @@ define([ verbose(message); allMessages.push(message); if (!initializing) { - if (PARANOIA) { - // FIXME this is out of sync with the application logic - onEvent(); - } + // FIXME this is out of sync with the application logic + console.log("xxx"); + window.cryptpad_propogate(); + } else { + console.log("init"); } realtime.message(message); if (/\[5,/.test(message)) { verbose("pong"); } @@ -281,16 +287,9 @@ define([ if (!initializing) { if (/\[2,/.test(message)) { //verbose("Got a patch"); - if (whoami.test(message)) { - //verbose("Received own message"); - } else { - //verbose("Received remote message"); - // obviously this is only going to get called if... XXX wat - if (config.onRemote) { config.onRemote({ - realtime: realtime - //realtime.getUserDoc() - }); } - } + +//TODO clean this all up + } } }); @@ -329,7 +328,7 @@ define([ }, 200); // TODO maybe push this out to the application layer. - bindAllEvents(null, doc, onEvent, false); + //bindAllEvents(null, doc, onEvent, false); // TODO rename 'sharejs.attach' to imply what we want to do var genOp = toReturn.propogate = sharejs.attach({ diff --git a/www/_socket/sharejs_textarea-transport-only.js b/www/_socket/sharejs_textarea-transport-only.js index 64d220051..910eca008 100644 --- a/www/_socket/sharejs_textarea-transport-only.js +++ b/www/_socket/sharejs_textarea-transport-only.js @@ -46,27 +46,28 @@ var attachTextarea = function(config) { var content = {}; // FIXME this is only necessary because we need to be able to update the - // textarea. This is being deprecated, however. Instead + // textarea. This is being deprecated, however. Instead var replaceText = function(newText) { content = newText; }; // *** remote -> local changes - ctx.onRemove(function(pos, length) { - replaceText(ctx.getUserDoc()); - }); - - ctx.onInsert(function(pos, text) { + ctx.onPatch(function(pos, length) { replaceText(ctx.getUserDoc()); }); + // propogate() return function (newContent) { if (newContent !== content) { applyChange(ctx, ctx.getUserDoc(), newContent); if (ctx.getUserDoc() !== newContent) { console.log("Expected that: `ctx.getUserDoc() === newContent`!"); } + console.log("1: " + ctx.Sha.hex_sha256(ctx.getUserDoc())); + return true; } + console.log("no change"); + return false; }; }; diff --git a/www/common/chainpad.js b/www/common/chainpad.js index c5854c7cf..2d45bf8ba 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -1163,6 +1163,7 @@ module.exports.create = function (userName, authToken, channelId, initialState, Common.assert(typeof(initialState) === 'string'); var realtime = ChainPad.create(userName, authToken, channelId, initialState, conf); return { + Sha: Sha, onPatch: enterChainPad(realtime, function (handler) { Common.assert(typeof(handler) === 'function'); realtime.patchHandlers.push(handler); diff --git a/www/pad/realtime-wysiwyg.js b/www/pad/realtime-wysiwyg.js index 30529a90f..2b35ef0a8 100644 --- a/www/pad/realtime-wysiwyg.js +++ b/www/pad/realtime-wysiwyg.js @@ -327,10 +327,11 @@ console.log(new Error().stack); error(false, 'realtime.getUserDoc() !== docText'); } }; - +var now = function () { return new Date().getTime(); }; var userDocBeforePatch; var incomingPatch = function () { if (isErrorState || initializing) { return; } + console.log("before patch " + now()); userDocBeforePatch = userDocBeforePatch || getFixedDocText(doc, ifr.contentWindow); if (PARANOIA && userDocBeforePatch !== getFixedDocText(doc, ifr.contentWindow)) { error(false, "userDocBeforePatch !== getFixedDocText(doc, ifr.contentWindow)"); @@ -339,6 +340,7 @@ console.log(new Error().stack); if (!op) { return; } attempt(HTMLPatcher.applyOp)( userDocBeforePatch, op, doc.body, Rangy, ifr.contentWindow); + console.log("after patch " + now()); }; realtime.onUserListChange(function (userList) { From f62ec85a4c672ae6f0a0a2bea29a88b1ccc7d0dc Mon Sep 17 00:00:00 2001 From: Caleb James DeLisle Date: Fri, 25 Mar 2016 14:14:19 +0100 Subject: [PATCH 09/69] Shuffled around some assertions and logs --- www/_socket/main.js | 9 ++++++--- www/_socket/realtime-input.js | 4 ---- www/_socket/sharejs_textarea-transport-only.js | 3 +-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index d41274a98..d59acd28f 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -197,6 +197,11 @@ define([ // build a dom from HJSON, diff, and patch the editor applyHjson(shjson); //updateDebugTextarea(shjson); + + var shjson2 = JSON.stringify(Convert.core.hyperjson.fromDOM(inner)); + if (shjson2 !== shjson) { + throw new Error("change after conversion"); + } }; var onInit = function (info) { @@ -260,9 +265,7 @@ define([ var rti = module.realtimeInput = window.rti = realtimeInput.start(realtimeOptions); var propogate = window.cryptpad_propogate = function () { - var hjson = Convert.core.hyperjson.fromDOM(inner, isNotMagicLine); - var shjson = JSON.stringify(hjson); - + var shjson = JSON.stringify(Convert.core.hyperjson.fromDOM(inner, isNotMagicLine)); if (!rti.propogate(shjson)) { return; } rti.onEvent(shjson); }; diff --git a/www/_socket/realtime-input.js b/www/_socket/realtime-input.js index 3d36217b1..e40a19085 100644 --- a/www/_socket/realtime-input.js +++ b/www/_socket/realtime-input.js @@ -211,7 +211,6 @@ define([ warn("realtime.getUserDoc() !== newText"); } //try{throw new Error();}catch(e){console.log(e.stack);} - console.log("2: " + realtime.Sha.hex_sha256(realtime.getUserDoc())); }; // pass your shiny new realtime into initialization functions @@ -276,10 +275,7 @@ define([ allMessages.push(message); if (!initializing) { // FIXME this is out of sync with the application logic - console.log("xxx"); window.cryptpad_propogate(); - } else { - console.log("init"); } realtime.message(message); if (/\[5,/.test(message)) { verbose("pong"); } diff --git a/www/_socket/sharejs_textarea-transport-only.js b/www/_socket/sharejs_textarea-transport-only.js index 910eca008..3e2ba7fc8 100644 --- a/www/_socket/sharejs_textarea-transport-only.js +++ b/www/_socket/sharejs_textarea-transport-only.js @@ -35,6 +35,7 @@ var applyChange = function(ctx, oldval, newval) { if (newval.length !== commonStart + commonEnd) { if (ctx.localChange) { ctx.localChange(true); } ctx.insert(commonStart, newval.slice(commonStart, newval.length - commonEnd)); + console.log("insert: " + newval.slice(commonStart, newval.length - commonEnd)); } }; @@ -63,10 +64,8 @@ var attachTextarea = function(config) { if (ctx.getUserDoc() !== newContent) { console.log("Expected that: `ctx.getUserDoc() === newContent`!"); } - console.log("1: " + ctx.Sha.hex_sha256(ctx.getUserDoc())); return true; } - console.log("no change"); return false; }; }; From 669bcc1935e60166345bdcadc566be16efc52205 Mon Sep 17 00:00:00 2001 From: Caleb James DeLisle Date: Fri, 25 Mar 2016 14:26:31 +0100 Subject: [PATCH 10/69] If there is a difference in the hjson then send a message back --- www/_socket/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index d59acd28f..c1e1833cc 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -200,7 +200,7 @@ define([ var shjson2 = JSON.stringify(Convert.core.hyperjson.fromDOM(inner)); if (shjson2 !== shjson) { - throw new Error("change after conversion"); + rti.propogate(shjson2); } }; From 03932d0169c9f49231a04d24830c7f602b6edb23 Mon Sep 17 00:00:00 2001 From: Caleb James DeLisle Date: Fri, 25 Mar 2016 14:52:44 +0100 Subject: [PATCH 11/69] small changes to chainpad for testing --- www/common/chainpad.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/www/common/chainpad.js b/www/common/chainpad.js index 2d45bf8ba..3dd9382fc 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -369,7 +369,7 @@ var random = Patch.random = function (doc, opCount) { * along with this program. If not, see . */ -var PARANOIA = module.exports.PARANOIA = false; +var PARANOIA = module.exports.PARANOIA = true; /* throw errors over non-compliant messages which would otherwise be treated as invalid */ var TESTING = module.exports.TESTING = false; @@ -999,13 +999,14 @@ var handleMessage = ChainPad.handleMessage = function (realtime, msgStr) { && Common.strcmp(realtime.best.hashOf, msg.hashOf) > 0)) { // switch chains + debug(realtime, "Patch [" + msg.hashOf + "] is best, switching chains."); while (commonAncestor && !isAncestorOf(realtime, commonAncestor, msg)) { toRevert.push(commonAncestor); commonAncestor = getParent(realtime, commonAncestor); } Common.assert(commonAncestor); } else { - debug(realtime, "Patch [" + msg.hashOf + "] chain is ["+pcMsg+"] best chain is ["+pcBest+"]"); + debug(realtime, "Patch [" + msg.hashOf + "] chain not best, staying."); if (Common.PARANOIA) { check(realtime); } return; } @@ -1025,6 +1026,9 @@ var handleMessage = ChainPad.handleMessage = function (realtime, msgStr) { for (var i = 0; i < toRevert.length; i++) { authDocAtTimeOfPatch = Patch.apply(toRevert[i].content.inverseOf, authDocAtTimeOfPatch); + if (Common.PARANOIA) { + Common.assert(Sha.hex_sha256(authDocAtTimeOfPatch) === toRevert[i].content.parentHash); + } } // toApply.length-1 because we do not want to apply the new patch. @@ -1033,6 +1037,9 @@ var handleMessage = ChainPad.handleMessage = function (realtime, msgStr) { toApply[i].content.inverseOf = Patch.invert(toApply[i].content, authDocAtTimeOfPatch); toApply[i].content.inverseOf.inverseOf = toApply[i].content; } + if (Common.PARANOIA) { + Common.assert(Sha.hex_sha256(authDocAtTimeOfPatch) === toApply[i].content.parentHash); + } authDocAtTimeOfPatch = Patch.apply(toApply[i].content, authDocAtTimeOfPatch); } From dbf31798d5bc930c3cd860c24ba20cf19060a6d3 Mon Sep 17 00:00:00 2001 From: Caleb James DeLisle Date: Fri, 25 Mar 2016 15:01:17 +0100 Subject: [PATCH 12/69] json-ot triggering PARANOIA errors in ChainPad --- www/common/json-ot.js | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/www/common/json-ot.js b/www/common/json-ot.js index 238ccc18a..37172b95c 100644 --- a/www/common/json-ot.js +++ b/www/common/json-ot.js @@ -5,19 +5,23 @@ define([ var JsonOT = {}; var validate = JsonOT.validate = function (text, toTransform, transformBy) { - var resultOp = ChainPad.Operation.transform0(text, toTransform, transformBy); - var text2 = ChainPad.Operation.apply(transformBy, text); - var text3 = ChainPad.Operation.apply(resultOp, text2); try { - JSON.parse(text3); - return resultOp; - } catch (e) { - console.error(e); - console.log({ - resultOp: resultOp, - text2: text2, - text3: text3 - }); + var resultOp = ChainPad.Operation.transform0(text, toTransform, transformBy); + var text2 = ChainPad.Operation.apply(transformBy, text); + var text3 = ChainPad.Operation.apply(resultOp, text2); + try { + JSON.parse(text3); + return resultOp; + } catch (e) { + console.error(e); + console.log({ + resultOp: resultOp, + text2: text2, + text3: text3 + }); + } + } catch (x) { + console.error(x); } // returning **null** breaks out of the loop From 62eabbc7aef94cbc3d9a4b9b54aa7a8dcf2fe9da Mon Sep 17 00:00:00 2001 From: Caleb James DeLisle Date: Fri, 25 Mar 2016 15:23:19 +0100 Subject: [PATCH 13/69] If a message does not match parent hash, don't delete it from storage --- www/common/chainpad.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/www/common/chainpad.js b/www/common/chainpad.js index 3dd9382fc..15b5505c6 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -369,7 +369,7 @@ var random = Patch.random = function (doc, opCount) { * along with this program. If not, see . */ -var PARANOIA = module.exports.PARANOIA = true; +var PARANOIA = module.exports.PARANOIA = false; /* throw errors over non-compliant messages which would otherwise be treated as invalid */ var TESTING = module.exports.TESTING = false; @@ -1046,8 +1046,7 @@ var handleMessage = ChainPad.handleMessage = function (realtime, msgStr) { if (Sha.hex_sha256(authDocAtTimeOfPatch) !== patch.parentHash) { debug(realtime, "patch [" + msg.hashOf + "] parentHash is not valid"); if (Common.PARANOIA) { check(realtime); } - if (Common.TESTING) { throw new Error(); } - delete realtime.messages[msg.hashOf]; + //delete realtime.messages[msg.hashOf]; return; } From 12dcbc91218fbf6f46297b9cc4271d352b55bd96 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 25 Mar 2016 16:14:17 +0100 Subject: [PATCH 14/69] fix quick red fox and make test.cancel work again --- www/_socket/main.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index c1e1833cc..22a403720 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -40,7 +40,7 @@ define([ var setRandomizedInterval = function (func, target, range) { var timeout; var again = function () { - setTimeout(function () { + timeout = setTimeout(function () { again(); func(); }, target - (range / 2) + Math.random() * range); @@ -273,13 +273,13 @@ define([ var testInput = window.testInput = function (el, offset) { var i = 0, j = offset, - input = "The quick red fox jumped over the lazy brown dog. ", + input = "The quick red fox jumps over the lazy brown dog. ", l = input.length, errors = 0, max_errors = 15, interval; var cancel = function () { - //if (interval) { interval.cancel(); } + if (interval) { interval.cancel(); } }; interval = setRandomizedInterval(function () { From 98c85cef8b0ddf2f4d40a6ef1aa51a03fa7ea3f2 Mon Sep 17 00:00:00 2001 From: Caleb James DeLisle Date: Fri, 25 Mar 2016 16:49:27 +0100 Subject: [PATCH 15/69] xxx --- www/_socket/main.js | 3 --- www/_socket/realtime-input.js | 2 +- www/common/chainpad.js | 6 +++--- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index c1e1833cc..71e3a4d39 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -174,14 +174,11 @@ define([ // apply patches, and try not to lose the cursor in the process! var applyHjson = function (shjson) { //setEditable(false); - console.log(now()); var userDocStateDom = Convert.hjson.to.dom(JSON.parse(shjson)); userDocStateDom.setAttribute("contenteditable", "true"); // lol wtf - console.log(now()); //assertStateMatches(); var patch = (DD).diff(inner, userDocStateDom); (DD).apply(inner, patch); - console.log(now()); // push back to the textarea so we get a userDocState //setEditable(true); }; diff --git a/www/_socket/realtime-input.js b/www/_socket/realtime-input.js index e40a19085..289f3d348 100644 --- a/www/_socket/realtime-input.js +++ b/www/_socket/realtime-input.js @@ -172,7 +172,7 @@ define([ // wrap up the reconnecting websocket with our additional stack logic var socket = makeWebsocket(websocketUrl); - var allMessages = []; + var allMessages = window.chainpad_allMessages = []; var isErrorState = false; var initializing = true; var recoverableErrorCount = 0; diff --git a/www/common/chainpad.js b/www/common/chainpad.js index 15b5505c6..e555cd55b 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -1033,13 +1033,13 @@ var handleMessage = ChainPad.handleMessage = function (realtime, msgStr) { // toApply.length-1 because we do not want to apply the new patch. for (var i = 0; i < toApply.length-1; i++) { + if (Common.PARANOIA) { + Common.assert(Sha.hex_sha256(authDocAtTimeOfPatch) === toApply[i].content.parentHash); + } if (typeof(toApply[i].content.inverseOf) === 'undefined') { toApply[i].content.inverseOf = Patch.invert(toApply[i].content, authDocAtTimeOfPatch); toApply[i].content.inverseOf.inverseOf = toApply[i].content; } - if (Common.PARANOIA) { - Common.assert(Sha.hex_sha256(authDocAtTimeOfPatch) === toApply[i].content.parentHash); - } authDocAtTimeOfPatch = Patch.apply(toApply[i].content, authDocAtTimeOfPatch); } From b372b0b77c8f6f8290332d39cf0466fd5197a670 Mon Sep 17 00:00:00 2001 From: Caleb James DeLisle Date: Fri, 25 Mar 2016 17:35:07 +0100 Subject: [PATCH 16/69] small change to chainpad in order to make it more likely to fail if the authDoc goes into the wrong state --- www/common/chainpad.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/www/common/chainpad.js b/www/common/chainpad.js index e555cd55b..161463654 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -1043,12 +1043,18 @@ var handleMessage = ChainPad.handleMessage = function (realtime, msgStr) { authDocAtTimeOfPatch = Patch.apply(toApply[i].content, authDocAtTimeOfPatch); } - if (Sha.hex_sha256(authDocAtTimeOfPatch) !== patch.parentHash) { + var authDocHashAtTimeOfPatch = Sha.hex_sha256(authDocAtTimeOfPatch); + if (authDocHashAtTimeOfPatch !== patch.parentHash) { debug(realtime, "patch [" + msg.hashOf + "] parentHash is not valid"); if (Common.PARANOIA) { check(realtime); } //delete realtime.messages[msg.hashOf]; return; } + if (authDocAtTimeOfPatch === realtime.authDoc && + authDocHashAtTimeOfPatch !== realtime.best.inverseOf.parentHash) + { + throw new Error("authDoc does not match chain head"); + } var simplePatch = Patch.simplify(patch, authDocAtTimeOfPatch, realtime.config.operationSimplify); From e26246178f2cdfdad255f277a3e9d7377839935d Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 25 Mar 2016 18:01:23 +0100 Subject: [PATCH 17/69] start to clean up and give things more sensible names. get rid of the textarea entirely --- www/_socket/index.html | 13 +------------ www/_socket/main.js | 14 +++++++------- www/_socket/realtime-input.js | 7 +++---- ..._textarea-transport-only.js => text-patcher.js} | 7 ++----- 4 files changed, 13 insertions(+), 28 deletions(-) rename www/_socket/{sharejs_textarea-transport-only.js => text-patcher.js} (96%) diff --git a/www/_socket/index.html b/www/_socket/index.html index 6ee2a8596..830d56125 100644 --- a/www/_socket/index.html +++ b/www/_socket/index.html @@ -14,28 +14,17 @@ left:0px; bottom:0px; right:0px; - width:70%; + width:100%; height:100%; border:none; margin:0; padding:0; overflow:hidden; } - #feedback { - position: fixed; - top: 0px; - right: 0px; - border: 0px; - height: 100vh; - width: 30vw; - background-color: #222; - color: #ccc; - } - diff --git a/www/_socket/main.js b/www/_socket/main.js index 86ccdc4a1..43dd1d824 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -90,7 +90,7 @@ define([ var $textarea = $('#feedback'); var setEditable = function (bool) { - // inner.style.backgroundColor = bool? 'unset': 'grey'; + inner.style.backgroundColor = bool? 'unset': 'grey'; inner.setAttribute('contenteditable', bool); }; @@ -197,7 +197,7 @@ define([ var shjson2 = JSON.stringify(Convert.core.hyperjson.fromDOM(inner)); if (shjson2 !== shjson) { - rti.propogate(shjson2); + rti.patchText(shjson2); } }; @@ -220,8 +220,8 @@ define([ var onAbort = function (info) { console.log("Aborting the session!"); // stop the user from continuing to edit + // by setting the editable to false setEditable(false); - // TODO inform them that the session was torn down toolbar.failed(); }; @@ -246,7 +246,6 @@ define([ // reject patch if it results in invalid JSON transformFunction : JsonOT.validate, - // websocketURL, ofc websocketURL: Config.websocketURL, // username @@ -259,15 +258,16 @@ define([ cryptKey: key.cryptKey }; - var rti = module.realtimeInput = window.rti = realtimeInput.start(realtimeOptions); + var rti = module.realtimeInput = realtimeInput.start(realtimeOptions); + // FIXME Spaghetti code. realtime-input needs access to this variable.. var propogate = window.cryptpad_propogate = function () { var shjson = JSON.stringify(Convert.core.hyperjson.fromDOM(inner, isNotMagicLine)); - if (!rti.propogate(shjson)) { return; } + if (!rti.patchText(shjson)) { return; } rti.onEvent(shjson); }; - var testInput = window.testInput = function (el, offset) { + var testInput = function (el, offset) { var i = 0, j = offset, input = "The quick red fox jumps over the lazy brown dog. ", diff --git a/www/_socket/realtime-input.js b/www/_socket/realtime-input.js index 289f3d348..926fc2e1d 100644 --- a/www/_socket/realtime-input.js +++ b/www/_socket/realtime-input.js @@ -19,10 +19,10 @@ define([ '/bower_components/reconnectingWebsocket/reconnecting-websocket.js', '/common/crypto.js', '/_socket/toolbar.js', - '/_socket/sharejs_textarea-transport-only.js', + '/_socket/text-patcher.js', '/common/chainpad.js', '/bower_components/jquery/dist/jquery.min.js', -], function (Messages,/*FIXME*/ ReconnectingWebSocket, Crypto, Toolbar, sharejs) { +], function (Messages,/*FIXME*/ ReconnectingWebSocket, Crypto, Toolbar, TextPatcher) { var $ = window.jQuery; var ChainPad = window.ChainPad; var PARANOIA = true; @@ -326,8 +326,7 @@ define([ // TODO maybe push this out to the application layer. //bindAllEvents(null, doc, onEvent, false); - // TODO rename 'sharejs.attach' to imply what we want to do - var genOp = toReturn.propogate = sharejs.attach({ + toReturn.patchText = TextPatcher.create({ realtime: realtime }); diff --git a/www/_socket/sharejs_textarea-transport-only.js b/www/_socket/text-patcher.js similarity index 96% rename from www/_socket/sharejs_textarea-transport-only.js rename to www/_socket/text-patcher.js index 3e2ba7fc8..bcbadb2e9 100644 --- a/www/_socket/sharejs_textarea-transport-only.js +++ b/www/_socket/text-patcher.js @@ -7,9 +7,6 @@ define(function () { it performs a diff on the two values, and generates patches which are then passed into `ctx.remove` and `ctx.insert` - - - */ var applyChange = function(ctx, oldval, newval) { // Strings are immutable and have reference equality. I think this test is O(1), so its worth doing. @@ -39,7 +36,7 @@ var applyChange = function(ctx, oldval, newval) { } }; -var attachTextarea = function(config) { +var create = function(config) { var ctx = config.realtime; // initial state will always fail the !== check in genop. @@ -70,5 +67,5 @@ var attachTextarea = function(config) { }; }; -return { attach: attachTextarea }; +return { create: create }; }); From 478ccbf984ab351a14d15949d6486bd3d5ec70f9 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 11:04:34 +0200 Subject: [PATCH 18/69] revert changes to chainpad --- www/common/chainpad.js | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/www/common/chainpad.js b/www/common/chainpad.js index 161463654..c5854c7cf 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -999,14 +999,13 @@ var handleMessage = ChainPad.handleMessage = function (realtime, msgStr) { && Common.strcmp(realtime.best.hashOf, msg.hashOf) > 0)) { // switch chains - debug(realtime, "Patch [" + msg.hashOf + "] is best, switching chains."); while (commonAncestor && !isAncestorOf(realtime, commonAncestor, msg)) { toRevert.push(commonAncestor); commonAncestor = getParent(realtime, commonAncestor); } Common.assert(commonAncestor); } else { - debug(realtime, "Patch [" + msg.hashOf + "] chain not best, staying."); + debug(realtime, "Patch [" + msg.hashOf + "] chain is ["+pcMsg+"] best chain is ["+pcBest+"]"); if (Common.PARANOIA) { check(realtime); } return; } @@ -1026,16 +1025,10 @@ var handleMessage = ChainPad.handleMessage = function (realtime, msgStr) { for (var i = 0; i < toRevert.length; i++) { authDocAtTimeOfPatch = Patch.apply(toRevert[i].content.inverseOf, authDocAtTimeOfPatch); - if (Common.PARANOIA) { - Common.assert(Sha.hex_sha256(authDocAtTimeOfPatch) === toRevert[i].content.parentHash); - } } // toApply.length-1 because we do not want to apply the new patch. for (var i = 0; i < toApply.length-1; i++) { - if (Common.PARANOIA) { - Common.assert(Sha.hex_sha256(authDocAtTimeOfPatch) === toApply[i].content.parentHash); - } if (typeof(toApply[i].content.inverseOf) === 'undefined') { toApply[i].content.inverseOf = Patch.invert(toApply[i].content, authDocAtTimeOfPatch); toApply[i].content.inverseOf.inverseOf = toApply[i].content; @@ -1043,18 +1036,13 @@ var handleMessage = ChainPad.handleMessage = function (realtime, msgStr) { authDocAtTimeOfPatch = Patch.apply(toApply[i].content, authDocAtTimeOfPatch); } - var authDocHashAtTimeOfPatch = Sha.hex_sha256(authDocAtTimeOfPatch); - if (authDocHashAtTimeOfPatch !== patch.parentHash) { + if (Sha.hex_sha256(authDocAtTimeOfPatch) !== patch.parentHash) { debug(realtime, "patch [" + msg.hashOf + "] parentHash is not valid"); if (Common.PARANOIA) { check(realtime); } - //delete realtime.messages[msg.hashOf]; + if (Common.TESTING) { throw new Error(); } + delete realtime.messages[msg.hashOf]; return; } - if (authDocAtTimeOfPatch === realtime.authDoc && - authDocHashAtTimeOfPatch !== realtime.best.inverseOf.parentHash) - { - throw new Error("authDoc does not match chain head"); - } var simplePatch = Patch.simplify(patch, authDocAtTimeOfPatch, realtime.config.operationSimplify); @@ -1175,7 +1163,6 @@ module.exports.create = function (userName, authToken, channelId, initialState, Common.assert(typeof(initialState) === 'string'); var realtime = ChainPad.create(userName, authToken, channelId, initialState, conf); return { - Sha: Sha, onPatch: enterChainPad(realtime, function (handler) { Common.assert(typeof(handler) === 'function'); realtime.patchHandlers.push(handler); From 5591aae8fa7fbf2503e970323d2a6bb818867d9d Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 11:12:46 +0200 Subject: [PATCH 19/69] Clean up main file * convert.js includes the vdom library, which we aren't using anymore - removed, and replaced with the simple functions from Hyperjson and Hyperscript * removed several variables that had been exported to 'window' * moved the testing functions out into their own file for easier reuse * restructured realtime initialization to be more compact --- www/_socket/main.js | 173 +++++++++++--------------------------------- 1 file changed, 44 insertions(+), 129 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index 43dd1d824..5e504bb47 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -3,22 +3,24 @@ define([ '/common/messages.js', '/common/crypto.js', '/_socket/realtime-input.js', - '/common/convert.js', + '/common/hyperjson.js', + '/common/hyperscript.js', '/_socket/toolbar.js', '/common/cursor.js', '/common/json-ot.js', + '/_socket/typingTest.js', '/bower_components/diff-dom/diffDOM.js', '/bower_components/jquery/dist/jquery.min.js', '/customize/pad.js' -], function (Config, Messages, Crypto, realtimeInput, Convert, Toolbar, Cursor, JsonOT) { +], function (Config, Messages, Crypto, realtimeInput, Hyperjson, Hyperscript, Toolbar, Cursor, JsonOT, TypingTest) { var $ = window.jQuery; var ifrw = $('#pad-iframe')[0].contentWindow; var Ckeditor; // to be initialized later... var DiffDom = window.diffDOM; - window.Convert = Convert; - - window.Toolbar = Toolbar; + var hjsonToDom = function (H) { + return Hyperjson.callOn(H, Hyperscript); + }; var userName = Crypto.rand64(8), toolbar; @@ -37,25 +39,6 @@ define([ return true; }; - var setRandomizedInterval = function (func, target, range) { - var timeout; - var again = function () { - timeout = setTimeout(function () { - again(); - func(); - }, target - (range / 2) + Math.random() * range); - }; - again(); - return { - cancel: function () { - if (timeout) { - clearTimeout(timeout); - timeout = undefined; - } - } - }; - } - var andThen = function (Ckeditor) { $(window).on('hashchange', function() { window.location.reload(); @@ -87,10 +70,12 @@ define([ var inner = window.inner = documentBody; var cursor = window.cursor = Cursor(inner); - var $textarea = $('#feedback'); - var setEditable = function (bool) { - inner.style.backgroundColor = bool? 'unset': 'grey'; + // careful about putting attributes onto the DOM + // they get put into the chain, and you can have trouble + // getting rid of them later + + //inner.style.backgroundColor = bool? 'white': 'grey'; inner.setAttribute('contenteditable', bool); }; @@ -148,42 +133,44 @@ define([ } }; - var initializing = true; + var now = function () { return new Date().getTime() }; - var assertStateMatches = function () { - var userDocState = module.realtimeInput.realtime.getUserDoc(); - var currentState = $textarea.val(); - if (currentState !== userDocState) { - console.log({ - userDocState: userDocState, - currentState: currentState - }); - throw new Error("currentState !== userDocState"); - } - }; + var realtimeOptions = { + // configuration :D + doc: inner, - var updateDebugTextarea = function (shjson) { - window.setTimeout(function () { - $textarea.val(shjson); - }, 0); - }; + // provide initialstate... + initialState: JSON.stringify(Hyperjson.fromDOM(inner, isNotMagicLine)), - var now = function () { return new Date().getTime() }; + // really basic operational transform + // reject patch if it results in invalid JSON + transformFunction : JsonOT.validate, + + websocketURL: Config.websocketURL, + + // username + userName: userName, + + // communication channel name + channel: key.channel, + + // encryption key + cryptKey: key.cryptKey + }; var DD = new DiffDom(diffOptions); + // apply patches, and try not to lose the cursor in the process! var applyHjson = function (shjson) { - //setEditable(false); - var userDocStateDom = Convert.hjson.to.dom(JSON.parse(shjson)); + var userDocStateDom = hjsonToDom(JSON.parse(shjson)); userDocStateDom.setAttribute("contenteditable", "true"); // lol wtf - //assertStateMatches(); var patch = (DD).diff(inner, userDocStateDom); (DD).apply(inner, patch); - // push back to the textarea so we get a userDocState - //setEditable(true); }; - var onRemote = function (info) { + var initializing = true; + + var onRemote = realtimeOptions.onRemote = function (info) { if (initializing) { return; } var shjson = info.realtime.getUserDoc(); @@ -193,21 +180,20 @@ define([ // build a dom from HJSON, diff, and patch the editor applyHjson(shjson); - //updateDebugTextarea(shjson); - var shjson2 = JSON.stringify(Convert.core.hyperjson.fromDOM(inner)); + var shjson2 = JSON.stringify(Hyperjson.fromDOM(inner)); if (shjson2 !== shjson) { rti.patchText(shjson2); } }; - var onInit = function (info) { + var onInit = realtimeOptions.onInit = function (info) { var $bar = $('#pad-iframe')[0].contentWindow.$('#cke_1_toolbox'); toolbar = info.realtime.toolbar = Toolbar.create($bar, userName, info.realtime); /* TODO handle disconnects and such*/ }; - var onReady = function (info) { + var onReady = realtimeOptions.onReady = function (info) { console.log("Unlocking editor"); initializing = false; setEditable(true); @@ -217,7 +203,7 @@ define([ applyHjson(shjson); }; - var onAbort = function (info) { + var onAbort = realtimeOptions.onAbort = function (info) { console.log("Aborting the session!"); // stop the user from continuing to edit // by setting the editable to false @@ -225,90 +211,19 @@ define([ toolbar.failed(); }; - var realtimeOptions = { - // configuration :D - doc: inner, - // first thing called - onInit: onInit, - - onReady: onReady, - - // when remote changes occur - onRemote: onRemote, - - // handle aborts - onAbort: onAbort, - - // provide initialstate... - initialState: JSON.stringify(Convert.core.hyperjson.fromDOM(inner, isNotMagicLine)), - - // really basic operational transform - // reject patch if it results in invalid JSON - transformFunction : JsonOT.validate, - - websocketURL: Config.websocketURL, - - // username - userName: userName, - - // communication channel name - channel: key.channel, - - // encryption key - cryptKey: key.cryptKey - }; - var rti = module.realtimeInput = realtimeInput.start(realtimeOptions); // FIXME Spaghetti code. realtime-input needs access to this variable.. var propogate = window.cryptpad_propogate = function () { - var shjson = JSON.stringify(Convert.core.hyperjson.fromDOM(inner, isNotMagicLine)); + var shjson = JSON.stringify(Hyperjson.fromDOM(inner, isNotMagicLine)); if (!rti.patchText(shjson)) { return; } rti.onEvent(shjson); }; - var testInput = function (el, offset) { - var i = 0, - j = offset, - input = "The quick red fox jumps over the lazy brown dog. ", - l = input.length, - errors = 0, - max_errors = 15, - interval; - var cancel = function () { - if (interval) { interval.cancel(); } - }; - - interval = setRandomizedInterval(function () { - propogate(); - try { - el.replaceData(j, 0, input.charAt(i)); - } catch (err) { - errors++; - if (errors >= max_errors) { - console.log("Max error number exceeded"); - cancel(); - } - - console.error(err); - var next = document.createTextNode("-"); - window.inner.appendChild(next); - el = next; - j = -1; - } - i = (i + 1) % l; - j++; - }, 200, 50); - - return { - cancel: cancel - }; - }; - var easyTest = window.easyTest = function () { cursor.update(); var start = cursor.Range.start; - var test = testInput(start.el, start.offset); + var test = TypingTest.testInput(start.el, start.offset, propogate); propogate(); return test; }; From d852c578d833f1120622da37a80ba5a1a897b85d Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 11:16:13 +0200 Subject: [PATCH 20/69] removed dead code --- www/_socket/realtime-input.js | 65 +---------------------------------- 1 file changed, 1 insertion(+), 64 deletions(-) diff --git a/www/_socket/realtime-input.js b/www/_socket/realtime-input.js index 926fc2e1d..4208bc805 100644 --- a/www/_socket/realtime-input.js +++ b/www/_socket/realtime-input.js @@ -46,49 +46,7 @@ define([ console.error(x); recoverableErrors++; if (recoverableErrors >= MAX_RECOVERABLE_ERRORS) { - alert("FAIL"); - } - }; - - // ------------------ Trapping Keyboard Events ---------------------- // - - var _unused_bindEvents = function (element, events, callback, unbind) { - for (var i = 0; i < events.length; i++) { - var e = events[i]; - if (element.addEventListener) { - if (unbind) { - element.removeEventListener(e, callback, false); - } else { - element.addEventListener(e, callback, false); - } - } else { - if (unbind) { - element.detachEvent('on' + e, callback); - } else { - element.attachEvent('on' + e, callback); - } - } - } - }; - - var _unused_bindAllEvents = function (textarea, docBody, onEvent, unbind) - { - /* - we use docBody for the purposes of CKEditor. - because otherwise special keybindings like ctrl-b and ctrl-i - would open bookmarks and info instead of applying bold/italic styles - */ - if (docBody) { - bindEvents(docBody, - ['textInput', 'keydown', 'keyup', 'select', 'cut', 'paste'], - onEvent, - unbind); - } - if (textarea) { - bindEvents(textarea, - ['mousedown','mouseup','click','change'], - onEvent, - unbind); + window.alert("FAIL"); } }; @@ -180,14 +138,6 @@ define([ var toReturn = { socket: socket }; socket.onOpen.push(function (evt) { - if (!initializing) { - console.log("Starting"); - // realtime is passed around as an attribute of the socket - // FIXME?? - socket.realtime.start(); - return; - } - var realtime = toReturn.realtime = socket.realtime = // everybody has a username, and we assume they don't collide // usernames are used to determine whether a message is remote @@ -278,16 +228,6 @@ define([ window.cryptpad_propogate(); } realtime.message(message); - if (/\[5,/.test(message)) { verbose("pong"); } - - if (!initializing) { - if (/\[2,/.test(message)) { - //verbose("Got a patch"); - -//TODO clean this all up - - } - } }); // actual socket bindings @@ -323,9 +263,6 @@ define([ } // it's working as expected, continue }, 200); - // TODO maybe push this out to the application layer. - //bindAllEvents(null, doc, onEvent, false); - toReturn.patchText = TextPatcher.create({ realtime: realtime }); From 941f5361ea950e2176d8356028f852d60f73502c Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 11:16:50 +0200 Subject: [PATCH 21/69] forgot to add 'typingTest', which main depends on --- www/_socket/typingTest.js | 64 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 www/_socket/typingTest.js diff --git a/www/_socket/typingTest.js b/www/_socket/typingTest.js new file mode 100644 index 000000000..4edc9d142 --- /dev/null +++ b/www/_socket/typingTest.js @@ -0,0 +1,64 @@ +define(function () { + var setRandomizedInterval = function (func, target, range) { + var timeout; + var again = function () { + timeout = setTimeout(function () { + again(); + func(); + }, target - (range / 2) + Math.random() * range); + }; + again(); + return { + cancel: function () { + if (timeout) { + clearTimeout(timeout); + timeout = undefined; + } + } + }; + }; + + var testInput = function (el, offset, cb) { + var i = 0, + j = offset, + input = "The quick red fox jumps over the lazy brown dog. ", + l = input.length, + errors = 0, + max_errors = 15, + interval; + var cancel = function () { + if (interval) { interval.cancel(); } + }; + + interval = setRandomizedInterval(function () { + cb(); + try { + el.replaceData(j, 0, input.charAt(i)); + } catch (err) { + errors++; + if (errors >= max_errors) { + console.log("Max error number exceeded"); + cancel(); + } + + console.error(err); + var next = document.createTextNode(""); + window.inner.appendChild(next); + el = next; + j = 0; + return; + } + i = (i + 1) % l; + j++; + }, 200, 50); + + return { + cancel: cancel + }; + }; + + return { + testInput: testInput, + setRandomizedInterval: setRandomizedInterval + }; +}); From e699073d457c54b7a61629a57102230787df4a26 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 11:54:55 +0200 Subject: [PATCH 22/69] attempt to preserve the magic line plugin while someone else is typing --- www/_socket/main.js | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index 5e504bb47..1dc106ed4 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -84,12 +84,31 @@ define([ var diffOptions = { preDiffApply: function (info) { - /* TODO DiffDOM will filter out magicline plugin elements + /* DiffDOM will filter out magicline plugin elements in practice this will make it impossible to use it - while someone else is typing, which could be annoying + while someone else is typing, which could be annoying. we should check when such an element is going to be removed, and prevent that from happening. */ + if (info.node && info.node.tagName === 'SPAN' && + info.node.contentEditable === "true") { + // it seems to be a magicline plugin element... + if (info.diff.action === 'removeElement') { + // and you're about to remove it... + // this probably isn't what you want + + /* + I have never seen this in the console, but the + magic line is still getting removed on remote + edits. This suggests that it's getting removed + by something other than diffDom. + */ + console.log("preventing removal of the magic line!"); + + // return true to prevent diff application + return true; + } + } // no use trying to recover the cursor if it doesn't exist if (!cursor.exists()) { return; } From 29e24f556c72ee56ab7b7ad21ef0b92f7405bfb0 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 12:13:57 +0200 Subject: [PATCH 23/69] kill another window variable --- www/_socket/main.js | 12 ++++++++++-- www/_socket/realtime-input.js | 5 +++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index 1dc106ed4..9c18eeca8 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -232,8 +232,16 @@ define([ var rti = module.realtimeInput = realtimeInput.start(realtimeOptions); - // FIXME Spaghetti code. realtime-input needs access to this variable.. - var propogate = window.cryptpad_propogate = function () { + /* + It's incredibly important that you assign 'rti.onLocal' + It's used inside of realtimeInput to make sure that all changes + make it into chainpad. + + It's being assigned this way because it can't be passed in, and + and can't be easily returned from realtime input without making + the code less extensible. + */ + var propogate = rti.onLocal = function () { var shjson = JSON.stringify(Hyperjson.fromDOM(inner, isNotMagicLine)); if (!rti.patchText(shjson)) { return; } rti.onEvent(shjson); diff --git a/www/_socket/realtime-input.js b/www/_socket/realtime-input.js index 4208bc805..06caa1c5d 100644 --- a/www/_socket/realtime-input.js +++ b/www/_socket/realtime-input.js @@ -224,8 +224,9 @@ define([ verbose(message); allMessages.push(message); if (!initializing) { - // FIXME this is out of sync with the application logic - window.cryptpad_propogate(); + if (toReturn.onLocal) { + toReturn.onLocal(); + } } realtime.message(message); }); From 3aebf7d2c252dfc9dc54ffe471beff3ff0929110 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 12:35:23 +0200 Subject: [PATCH 24/69] minor changes to pass linting --- www/_socket/main.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index 9c18eeca8..a7682429c 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -152,7 +152,7 @@ define([ } }; - var now = function () { return new Date().getTime() }; + var now = function () { return new Date().getTime(); }; var realtimeOptions = { // configuration :D @@ -202,7 +202,7 @@ define([ var shjson2 = JSON.stringify(Hyperjson.fromDOM(inner)); if (shjson2 !== shjson) { - rti.patchText(shjson2); + module.realtimeInput.patchText(shjson2); } }; From e51635c4bbfc9809c816c256a87159fa7def584e Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 15:34:58 +0200 Subject: [PATCH 25/69] fix index error --- www/_socket/typingTest.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/www/_socket/typingTest.js b/www/_socket/typingTest.js index 4edc9d142..8ade1caf7 100644 --- a/www/_socket/typingTest.js +++ b/www/_socket/typingTest.js @@ -45,8 +45,7 @@ define(function () { var next = document.createTextNode(""); window.inner.appendChild(next); el = next; - j = 0; - return; + j = -1; } i = (i + 1) % l; j++; From bac0e0ff887e0fb5548d072fadd9421d80660d6f Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 15:35:40 +0200 Subject: [PATCH 26/69] implement hyperjson filtering --- www/common/hyperjson.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/www/common/hyperjson.js b/www/common/hyperjson.js index 31a2caf08..71ba71987 100644 --- a/www/common/hyperjson.js +++ b/www/common/hyperjson.js @@ -47,7 +47,7 @@ define([], function () { return x; }; - var DOM2HyperJSON = function(el, predicate){ + var DOM2HyperJSON = function(el, predicate, filter){ if(!el.tagName && el.nodeType === Node.TEXT_NODE){ return el.textContent; } @@ -111,12 +111,16 @@ define([], function () { i = 0; for(; i < el.childNodes.length; i++){ - children.push(DOM2HyperJSON(el.childNodes[i], predicate)); + children.push(DOM2HyperJSON(el.childNodes[i], predicate, filter)); } result.push(children.filter(isTruthy)); - return result; + if (filter) { + return filter(result); + } else { + return result; + } }; return { From afa1104d850e1bab3c931737b8361d54881bda1a Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 15:36:03 +0200 Subject: [PATCH 27/69] Pull the cursor out of bogus BR tarpits when it gets stuck --- www/common/cursor.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/www/common/cursor.js b/www/common/cursor.js index 069f896c7..8eff84374 100644 --- a/www/common/cursor.js +++ b/www/common/cursor.js @@ -373,6 +373,26 @@ define([ }; }; + cursor.brFix = function () { + cursor.update(); + var start = Range.start; + var end = Range.end; + if (!start.el) { return; } + + if (start.el === end.el && start.offset === end.offset) { + if (start.el.tagName === 'BR') { + // get the parent element, which ought to be a P. + var P = start.el.parentNode; + + [cursor.fixStart, cursor.fixEnd].forEach(function (f) { + f(P, 0); + }); + + cursor.fixSelection(cursor.makeSelection(), cursor.makeRange()); + } + } + }; + return cursor; }; }); From 6c340a65271820ef8852b6573f3819a141fe5c6f Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 16:53:40 +0200 Subject: [PATCH 28/69] chainpad testing = true, reject non-compliant messages --- www/common/chainpad.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/common/chainpad.js b/www/common/chainpad.js index c5854c7cf..b2a81138c 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -372,7 +372,7 @@ var random = Patch.random = function (doc, opCount) { var PARANOIA = module.exports.PARANOIA = false; /* throw errors over non-compliant messages which would otherwise be treated as invalid */ -var TESTING = module.exports.TESTING = false; +var TESTING = module.exports.TESTING = true; var assert = module.exports.assert = function (expr) { if (!expr) { throw new Error("Failed assertion"); } From 772ca5d30e3db1a611d3dd2f3447779d7e87ccae Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 17:01:57 +0200 Subject: [PATCH 29/69] comment out debugging line --- www/_socket/text-patcher.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/_socket/text-patcher.js b/www/_socket/text-patcher.js index bcbadb2e9..e0abd33e4 100644 --- a/www/_socket/text-patcher.js +++ b/www/_socket/text-patcher.js @@ -32,7 +32,7 @@ var applyChange = function(ctx, oldval, newval) { if (newval.length !== commonStart + commonEnd) { if (ctx.localChange) { ctx.localChange(true); } ctx.insert(commonStart, newval.slice(commonStart, newval.length - commonEnd)); - console.log("insert: " + newval.slice(commonStart, newval.length - commonEnd)); + //console.log("insert: " + newval.slice(commonStart, newval.length - commonEnd)); } }; From 22290590cbfd2ec256ec69da57c173cc07fc651f Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 17:02:56 +0200 Subject: [PATCH 30/69] don't rely on window scope in typingTest.js --- www/_socket/typingTest.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/www/_socket/typingTest.js b/www/_socket/typingTest.js index 8ade1caf7..af09c72c8 100644 --- a/www/_socket/typingTest.js +++ b/www/_socket/typingTest.js @@ -18,10 +18,10 @@ define(function () { }; }; - var testInput = function (el, offset, cb) { + var testInput = function (doc, el, offset, cb) { var i = 0, j = offset, - input = "The quick red fox jumps over the lazy brown dog. ", + input = " The quick red fox jumps over the lazy brown dog.", l = input.length, errors = 0, max_errors = 15, @@ -43,7 +43,7 @@ define(function () { console.error(err); var next = document.createTextNode(""); - window.inner.appendChild(next); + doc.appendChild(next); el = next; j = -1; } From e446a3645c129233a110b34745222b8bd21ac2af Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Mar 2016 17:21:02 +0200 Subject: [PATCH 31/69] don't send funny BR attributes over the wire. Properly initialize the typing test --- www/_socket/main.js | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index a7682429c..ca3f8c713 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -230,7 +230,13 @@ define([ toolbar.failed(); }; - var rti = module.realtimeInput = realtimeInput.start(realtimeOptions); + var rti = window.CRYPTPAD_REALTIME = module.realtimeInput = + realtimeInput.start(realtimeOptions); + + var brFilter = function (hj) { + if (hj[1].type === '_moz') { hj[1].type = undefined; } + return hj; + }; /* It's incredibly important that you assign 'rti.onLocal' @@ -242,15 +248,24 @@ define([ the code less extensible. */ var propogate = rti.onLocal = function () { - var shjson = JSON.stringify(Hyperjson.fromDOM(inner, isNotMagicLine)); + var shjson = JSON.stringify(Hyperjson.fromDOM(inner, isNotMagicLine, brFilter)); if (!rti.patchText(shjson)) { return; } - rti.onEvent(shjson); + //rti.onEvent(shjson); }; + /* hitting enter makes a new line, but places the cursor inside + of the
instead of the

. This makes it such that you + cannot type until you click, which is rather unnacceptable. + If the cursor is ever inside such a
, you probably want + to push it out to the parent element, which ought to be a + paragraph tag. This needs to be done on keydown, otherwise + the first such keypress will not be inserted into the P. */ + inner.addEventListener('keydown', cursor.brFix); + var easyTest = window.easyTest = function () { cursor.update(); var start = cursor.Range.start; - var test = TypingTest.testInput(start.el, start.offset, propogate); + var test = TypingTest.testInput(inner, start.el, start.offset, propogate); propogate(); return test; }; From 523df40d092f45e30e4ff6b899dfee31e00b2927 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 30 Mar 2016 14:36:11 +0200 Subject: [PATCH 32/69] Debugging concurrent typing: track whether there are local operations in progress, such that we can tell whether a remote change is interrupting the DOM's conversion to hjson. --- www/_socket/main.js | 49 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index ca3f8c713..0974a66f0 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -18,6 +18,8 @@ define([ var Ckeditor; // to be initialized later... var DiffDom = window.diffDOM; + window.Hyperjson = Hyperjson; + var hjsonToDom = function (H) { return Hyperjson.callOn(H, Hyperscript); }; @@ -25,7 +27,9 @@ define([ var userName = Crypto.rand64(8), toolbar; - var module = {}; + var module = window.REALTIME_MODULE = { + localChangeInProgress: 0 + }; var isNotMagicLine = function (el) { // factor as: @@ -179,12 +183,29 @@ define([ var DD = new DiffDom(diffOptions); + var localWorkInProgress = function (stage) { + if (module.localChangeInProgress) { + console.error("Applied a change while a local patch was in progress"); + alert("local work was interrupted at stage: " + stage); + //module.realtimeInput.onLocal(); + return true; + } + return false; + }; + // apply patches, and try not to lose the cursor in the process! var applyHjson = function (shjson) { + + localWorkInProgress(1); // check if this would interrupt local work + var userDocStateDom = hjsonToDom(JSON.parse(shjson)); + localWorkInProgress(2); // check again userDocStateDom.setAttribute("contenteditable", "true"); // lol wtf + localWorkInProgress(3); // check again var patch = (DD).diff(inner, userDocStateDom); + localWorkInProgress(4); // check again (DD).apply(inner, patch); + localWorkInProgress(5); // check again }; var initializing = true; @@ -192,6 +213,8 @@ define([ var onRemote = realtimeOptions.onRemote = function (info) { if (initializing) { return; } + localWorkInProgress(0); + var shjson = info.realtime.getUserDoc(); // remember where the cursor is @@ -202,6 +225,7 @@ define([ var shjson2 = JSON.stringify(Hyperjson.fromDOM(inner)); if (shjson2 !== shjson) { + console.error("shjson2 !== shjson"); module.realtimeInput.patchText(shjson2); } }; @@ -209,6 +233,7 @@ define([ var onInit = realtimeOptions.onInit = function (info) { var $bar = $('#pad-iframe')[0].contentWindow.$('#cke_1_toolbox'); toolbar = info.realtime.toolbar = Toolbar.create($bar, userName, info.realtime); + /* TODO handle disconnects and such*/ }; @@ -230,16 +255,15 @@ define([ toolbar.failed(); }; - var rti = window.CRYPTPAD_REALTIME = module.realtimeInput = - realtimeInput.start(realtimeOptions); + var rti = module.realtimeInput = realtimeInput.start(realtimeOptions); + /* catch `type="_moz"` before it goes over the wire */ var brFilter = function (hj) { if (hj[1].type === '_moz') { hj[1].type = undefined; } return hj; }; - /* - It's incredibly important that you assign 'rti.onLocal' + /* It's incredibly important that you assign 'rti.onLocal' It's used inside of realtimeInput to make sure that all changes make it into chainpad. @@ -248,9 +272,20 @@ define([ the code less extensible. */ var propogate = rti.onLocal = function () { + /* if the problem were a matter of external patches being + applied while a local patch were in progress, then we would + expect to be able to check and find + 'module.localChangeInProgress' with a non-zero value while + we were applying a remote change. + */ + module.localChangeInProgress += 1; var shjson = JSON.stringify(Hyperjson.fromDOM(inner, isNotMagicLine, brFilter)); - if (!rti.patchText(shjson)) { return; } - //rti.onEvent(shjson); + if (!rti.patchText(shjson)) { + module.localChangeInProgress -= 1; + return; + } + rti.onEvent(shjson); + module.localChangeInProgress -= 1; }; /* hitting enter makes a new line, but places the cursor inside From aaf7c777cc5d54acc253a0e14031bb8b56b7857f Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 30 Mar 2016 14:38:10 +0200 Subject: [PATCH 33/69] add debugging info to the textPatcher * kill dead code * add assertions * better logging for insertions and removals --- www/_socket/text-patcher.js | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/www/_socket/text-patcher.js b/www/_socket/text-patcher.js index e0abd33e4..9f51c07b5 100644 --- a/www/_socket/text-patcher.js +++ b/www/_socket/text-patcher.js @@ -25,14 +25,37 @@ var applyChange = function(ctx, oldval, newval) { commonEnd++; } + var result; + + /* throw some assertions in here before dropping patches into the realtime + + */ + if (oldval.length !== commonStart + commonEnd) { if (ctx.localChange) { ctx.localChange(true); } - ctx.remove(commonStart, oldval.length - commonStart - commonEnd); + result = oldval.length - commonStart - commonEnd; + ctx.remove(commonStart, result); + console.log('removal at position: %s, length: %s', commonStart, result); + console.log("remove: [" + oldval.slice(commonStart, commonStart + result ) + ']'); } if (newval.length !== commonStart + commonEnd) { if (ctx.localChange) { ctx.localChange(true); } - ctx.insert(commonStart, newval.slice(commonStart, newval.length - commonEnd)); - //console.log("insert: " + newval.slice(commonStart, newval.length - commonEnd)); + result = newval.slice(commonStart, newval.length - commonEnd); + ctx.insert(commonStart, result); + console.log("insert: [" + result + "]"); + } + + var userDoc; + try { + var userDoc = ctx.getUserDoc(); + JSON.parse(userDoc); + } catch (err) { + console.error('[textPatcherParseErr]'); + console.error(err); + window.REALTIME_MODULE.textPatcher_parseError = { + error: err, + userDoc: userDoc + }; } }; @@ -43,15 +66,9 @@ var create = function(config) { // because nothing will equal this object var content = {}; - // FIXME this is only necessary because we need to be able to update the - // textarea. This is being deprecated, however. Instead - var replaceText = function(newText) { - content = newText; - }; - // *** remote -> local changes ctx.onPatch(function(pos, length) { - replaceText(ctx.getUserDoc()); + content = ctx.getUserDoc() }); // propogate() From d5772c6315cf1547ced65e078c1ff264118c7bc0 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 30 Mar 2016 14:39:41 +0200 Subject: [PATCH 34/69] when json-ot produces json that fails to parse... export the relevant data to a window variable so we can inspect it better --- www/common/json-ot.js | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/www/common/json-ot.js b/www/common/json-ot.js index 37172b95c..41a628dc4 100644 --- a/www/common/json-ot.js +++ b/www/common/json-ot.js @@ -14,14 +14,36 @@ define([ return resultOp; } catch (e) { console.error(e); - console.log({ + var info = window.REALTIME_MODULE.ot_parseError = { + type: 'resultParseError', resultOp: resultOp, + + toTransform: toTransform, + transformBy: transformBy, + + text1: text, text2: text2, - text3: text3 - }); + text3: text3, + error: e + }; + console.log('Debugging info available at `window.REALTIME_MODULE.ot_parseError`'); } } catch (x) { console.error(x); + console.error(e); + var info = window.REALTIME_MODULE.ot_applyError = { + type: 'resultParseError', + resultOp: resultOp, + + toTransform: toTransform, + transformBy: transformBy, + + text1: text, + text2: text2, + text3: text3, + error: e + }; + console.log('Debugging info available at `window.REALTIME_MODULE.ot_applyError`'); } // returning **null** breaks out of the loop From 96e03fcfa4ab218208810011462247ff2caf8c26 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 30 Mar 2016 15:29:28 +0200 Subject: [PATCH 35/69] Use latest chainpad without mutations --- www/common/chainpad.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/www/common/chainpad.js b/www/common/chainpad.js index b2a81138c..502e502ab 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -1443,7 +1443,13 @@ var rebase = Operation.rebase = function (oldOp, newOp) { * @param transformBy an existing operation which also has the same base. * @return toTransform *or* null if the result is a no-op. */ -var transform0 = Operation.transform0 = function (text, toTransform, transformBy) { + +var transform0 = Operation.transform0 = function (text, toTransformOrig, transformByOrig) { + // Cloning the original transformations makes this algorithm such that it + // **DOES NOT MUTATE ANYMORE** + var toTransform = Operation.clone(toTransformOrig); + var transformBy = Operation.clone(transformByOrig); + if (toTransform.offset > transformBy.offset) { if (toTransform.offset > transformBy.offset + transformBy.toRemove) { // simple rebase From df78d284c4e0ab3b66fc51583bb76b1a6929b303 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 1 Apr 2016 10:44:23 +0200 Subject: [PATCH 36/69] initial state should be valid JSON so JSON parse doesn't fail --- www/_socket/realtime-input.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/_socket/realtime-input.js b/www/_socket/realtime-input.js index 06caa1c5d..565200aba 100644 --- a/www/_socket/realtime-input.js +++ b/www/_socket/realtime-input.js @@ -148,7 +148,7 @@ define([ channel, // the channel we're to connect to // initialState argument. (optional) - config.initialState || '', + config.initialState || '{}', // transform function (optional), which handles conflicts { transformFunction: config.transformFunction }); From c50690349b4cd40643530deeef5074527d5d41b2 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 1 Apr 2016 10:46:54 +0200 Subject: [PATCH 37/69] comments describing the role of each argument --- www/common/json-ot.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/www/common/json-ot.js b/www/common/json-ot.js index 41a628dc4..98e1af4ec 100644 --- a/www/common/json-ot.js +++ b/www/common/json-ot.js @@ -6,6 +6,11 @@ define([ var validate = JsonOT.validate = function (text, toTransform, transformBy) { try { + // text = O (mutual common ancestor) + // toTransform = A (the first incoming operation) + // transformBy = B (the second incoming operation) + // threeway merge (0, A, B) + var resultOp = ChainPad.Operation.transform0(text, toTransform, transformBy); var text2 = ChainPad.Operation.apply(transformBy, text); var text3 = ChainPad.Operation.apply(resultOp, text2); From 2691d855823e82f07aa7cb8a9e3fabbfe4bc616a Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 1 Apr 2016 10:51:47 +0200 Subject: [PATCH 38/69] use forked chainpad with assertions for _socket --- www/_socket/chainpad.js | 1558 +++++++++++++++++++++++++++++++++ www/_socket/realtime-input.js | 2 +- 2 files changed, 1559 insertions(+), 1 deletion(-) create mode 100644 www/_socket/chainpad.js diff --git a/www/_socket/chainpad.js b/www/_socket/chainpad.js new file mode 100644 index 000000000..ead48ee26 --- /dev/null +++ b/www/_socket/chainpad.js @@ -0,0 +1,1558 @@ +(function(){ +var r=function(){var e="function"==typeof require&&require,r=function(i,o,u){o||(o=0);var n=r.resolve(i,o),t=r.m[o][n];if(!t&&e){if(t=e(n))return t}else if(t&&t.c&&(o=t.c,n=t.m,t=r.m[o][t.m],!t))throw new Error('failed to require "'+n+'" from '+o);if(!t)throw new Error('failed to require "'+i+'" from '+u);return t.exports||(t.exports={},t.call(t.exports,t,t.exports,r.relative(n,o))),t.exports};return r.resolve=function(e,n){var i=e,t=e+".js",o=e+"/index.js";return r.m[n][t]&&t?t:r.m[n][o]&&o?o:i},r.relative=function(e,t){return function(n){if("."!=n.charAt(0))return r(n,t,e);var o=e.split("/"),f=n.split("/");o.pop();for(var i=0;i. + */ +var Common = require('./Common'); +var Operation = require('./Operation'); +var Sha = require('./SHA256'); + +var Patch = module.exports; + +var create = Patch.create = function (parentHash) { + return { + type: 'Patch', + operations: [], + parentHash: parentHash + }; +}; + +var check = Patch.check = function (patch, docLength_opt) { + Common.assert(patch.type === 'Patch'); + Common.assert(Array.isArray(patch.operations)); + Common.assert(/^[0-9a-f]{64}$/.test(patch.parentHash)); + for (var i = patch.operations.length - 1; i >= 0; i--) { + Operation.check(patch.operations[i], docLength_opt); + if (i > 0) { + Common.assert(!Operation.shouldMerge(patch.operations[i], patch.operations[i-1])); + } + if (typeof(docLength_opt) === 'number') { + docLength_opt += Operation.lengthChange(patch.operations[i]); + } + } +}; + +var toObj = Patch.toObj = function (patch) { + if (Common.PARANOIA) { check(patch); } + var out = new Array(patch.operations.length+1); + var i; + for (i = 0; i < patch.operations.length; i++) { + out[i] = Operation.toObj(patch.operations[i]); + } + out[i] = patch.parentHash; + return out; +}; + +var fromObj = Patch.fromObj = function (obj) { + Common.assert(Array.isArray(obj) && obj.length > 0); + var patch = create(); + var i; + for (i = 0; i < obj.length-1; i++) { + patch.operations[i] = Operation.fromObj(obj[i]); + } + patch.parentHash = obj[i]; + if (Common.PARANOIA) { check(patch); } + return patch; +}; + +var hash = function (text) { + return Sha.hex_sha256(text); +}; + +var addOperation = Patch.addOperation = function (patch, op) { + if (Common.PARANOIA) { + check(patch); + Operation.check(op); + } + for (var i = 0; i < patch.operations.length; i++) { + if (Operation.shouldMerge(patch.operations[i], op)) { + op = Operation.merge(patch.operations[i], op); + patch.operations.splice(i,1); + if (op === null) { + //console.log("operations cancelled eachother"); + return; + } + i--; + } else { + var out = Operation.rebase(patch.operations[i], op); + if (out === op) { + // op could not be rebased further, insert it here to keep the list ordered. + patch.operations.splice(i,0,op); + return; + } else { + op = out; + // op was rebased, try rebasing it against the next operation. + } + } + } + patch.operations.push(op); + if (Common.PARANOIA) { check(patch); } +}; + +var clone = Patch.clone = function (patch) { + if (Common.PARANOIA) { check(patch); } + var out = create(); + out.parentHash = patch.parentHash; + for (var i = 0; i < patch.operations.length; i++) { + out.operations[i] = Operation.clone(patch.operations[i]); + } + return out; +}; + +var merge = Patch.merge = function (oldPatch, newPatch) { + if (Common.PARANOIA) { + check(oldPatch); + check(newPatch); + } + oldPatch = clone(oldPatch); + for (var i = newPatch.operations.length-1; i >= 0; i--) { + addOperation(oldPatch, newPatch.operations[i]); + } + return oldPatch; +}; + +var apply = Patch.apply = function (patch, doc) +{ + if (Common.PARANOIA) { + check(patch); + Common.assert(typeof(doc) === 'string'); + Common.assert(Sha.hex_sha256(doc) === patch.parentHash); + } + var newDoc = doc; + for (var i = patch.operations.length-1; i >= 0; i--) { + newDoc = Operation.apply(patch.operations[i], newDoc); + } + return newDoc; +}; + +var lengthChange = Patch.lengthChange = function (patch) +{ + if (Common.PARANOIA) { check(patch); } + var out = 0; + for (var i = 0; i < patch.operations.length; i++) { + out += Operation.lengthChange(patch.operations[i]); + } + return out; +}; + +var invert = Patch.invert = function (patch, doc) +{ + if (Common.PARANOIA) { + check(patch); + Common.assert(typeof(doc) === 'string'); + Common.assert(Sha.hex_sha256(doc) === patch.parentHash); + } + var rpatch = create(); + var newDoc = doc; + for (var i = patch.operations.length-1; i >= 0; i--) { + rpatch.operations[i] = Operation.invert(patch.operations[i], newDoc); + newDoc = Operation.apply(patch.operations[i], newDoc); + } + for (var i = rpatch.operations.length-1; i >= 0; i--) { + for (var j = i - 1; j >= 0; j--) { + rpatch.operations[i].offset += rpatch.operations[j].toRemove; + rpatch.operations[i].offset -= rpatch.operations[j].toInsert.length; + } + } + rpatch.parentHash = Sha.hex_sha256(newDoc); + if (Common.PARANOIA) { check(rpatch); } + return rpatch; +}; + +var simplify = Patch.simplify = function (patch, doc, operationSimplify) +{ + if (Common.PARANOIA) { + check(patch); + Common.assert(typeof(doc) === 'string'); + Common.assert(Sha.hex_sha256(doc) === patch.parentHash); + } + operationSimplify = operationSimplify || Operation.simplify; + var spatch = create(patch.parentHash); + var newDoc = doc; + var outOps = []; + var j = 0; + for (var i = patch.operations.length-1; i >= 0; i--) { + outOps[j] = operationSimplify(patch.operations[i], newDoc, Operation.simplify); + if (outOps[j]) { + newDoc = Operation.apply(outOps[j], newDoc); + j++; + } + } + spatch.operations = outOps.reverse(); + if (!spatch.operations[0]) { + spatch.operations.shift(); + } + if (Common.PARANOIA) { + check(spatch); + } + return spatch; +}; + +var equals = Patch.equals = function (patchA, patchB) { + if (patchA.operations.length !== patchB.operations.length) { return false; } + for (var i = 0; i < patchA.operations.length; i++) { + if (!Operation.equals(patchA.operations[i], patchB.operations[i])) { return false; } + } + return true; +}; + +var transform = Patch.transform = function (origToTransform, transformBy, doc, transformFunction) { + if (Common.PARANOIA) { + check(origToTransform, doc.length); + check(transformBy, doc.length); + Common.assert(Sha.hex_sha256(doc) === origToTransform.parentHash); + } + Common.assert(origToTransform.parentHash === transformBy.parentHash); + var resultOfTransformBy = apply(transformBy, doc); + + toTransform = clone(origToTransform); + var text = doc; + for (var i = toTransform.operations.length-1; i >= 0; i--) { + text = Operation.apply(toTransform.operations[i], text); + for (var j = transformBy.operations.length-1; j >= 0; j--) { + toTransform.operations[i] = Operation.transform(text, + toTransform.operations[i], + transformBy.operations[j], + transformFunction); + if (!toTransform.operations[i]) { + break; + } + } + if (Common.PARANOIA && toTransform.operations[i]) { + Operation.check(toTransform.operations[i], resultOfTransformBy.length); + } + } + var out = create(transformBy.parentHash); + for (var i = toTransform.operations.length-1; i >= 0; i--) { + if (toTransform.operations[i]) { + addOperation(out, toTransform.operations[i]); + } + } + + out.parentHash = Sha.hex_sha256(resultOfTransformBy); + + if (Common.PARANOIA) { + check(out, resultOfTransformBy.length); + } + return out; +}; + +var random = Patch.random = function (doc, opCount) { + Common.assert(typeof(doc) === 'string'); + opCount = opCount || (Math.floor(Math.random() * 30) + 1); + var patch = create(Sha.hex_sha256(doc)); + var docLength = doc.length; + while (opCount-- > 0) { + var op = Operation.random(docLength); + docLength += Operation.lengthChange(op); + addOperation(patch, op); + } + check(patch); + return patch; +}; + +}, +"SHA256.js": function(module, exports, require){ +/* A JavaScript implementation of the Secure Hash Algorithm, SHA-256 + * Version 0.3 Copyright Angel Marin 2003-2004 - http://anmar.eu.org/ + * Distributed under the BSD License + * Some bits taken from Paul Johnston's SHA-1 implementation + */ +(function () { + var chrsz = 8; /* bits per input character. 8 - ASCII; 16 - Unicode */ + function safe_add (x, y) { + var lsw = (x & 0xFFFF) + (y & 0xFFFF); + var msw = (x >> 16) + (y >> 16) + (lsw >> 16); + return (msw << 16) | (lsw & 0xFFFF); + } + function S (X, n) {return ( X >>> n ) | (X << (32 - n));} + function R (X, n) {return ( X >>> n );} + function Ch(x, y, z) {return ((x & y) ^ ((~x) & z));} + function Maj(x, y, z) {return ((x & y) ^ (x & z) ^ (y & z));} + function Sigma0256(x) {return (S(x, 2) ^ S(x, 13) ^ S(x, 22));} + function Sigma1256(x) {return (S(x, 6) ^ S(x, 11) ^ S(x, 25));} + function Gamma0256(x) {return (S(x, 7) ^ S(x, 18) ^ R(x, 3));} + function Gamma1256(x) {return (S(x, 17) ^ S(x, 19) ^ R(x, 10));} + function newArray (n) { + var a = []; + for (;n>0;n--) { + a.push(undefined); + } + return a; + } + function core_sha256 (m, l) { + var K = [0x428A2F98,0x71374491,0xB5C0FBCF,0xE9B5DBA5,0x3956C25B,0x59F111F1,0x923F82A4,0xAB1C5ED5,0xD807AA98,0x12835B01,0x243185BE,0x550C7DC3,0x72BE5D74,0x80DEB1FE,0x9BDC06A7,0xC19BF174,0xE49B69C1,0xEFBE4786,0xFC19DC6,0x240CA1CC,0x2DE92C6F,0x4A7484AA,0x5CB0A9DC,0x76F988DA,0x983E5152,0xA831C66D,0xB00327C8,0xBF597FC7,0xC6E00BF3,0xD5A79147,0x6CA6351,0x14292967,0x27B70A85,0x2E1B2138,0x4D2C6DFC,0x53380D13,0x650A7354,0x766A0ABB,0x81C2C92E,0x92722C85,0xA2BFE8A1,0xA81A664B,0xC24B8B70,0xC76C51A3,0xD192E819,0xD6990624,0xF40E3585,0x106AA070,0x19A4C116,0x1E376C08,0x2748774C,0x34B0BCB5,0x391C0CB3,0x4ED8AA4A,0x5B9CCA4F,0x682E6FF3,0x748F82EE,0x78A5636F,0x84C87814,0x8CC70208,0x90BEFFFA,0xA4506CEB,0xBEF9A3F7,0xC67178F2]; + var HASH = [0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19]; + var W = newArray(64); + var a, b, c, d, e, f, g, h, i, j; + var T1, T2; + /* append padding */ + m[l >> 5] |= 0x80 << (24 - l % 32); + m[((l + 64 >> 9) << 4) + 15] = l; + for ( var i = 0; i>5] |= (str.charCodeAt(i / chrsz) & mask) << (24 - i%32); + return bin; + } + function binb2hex (binarray) { + var hexcase = 0; /* hex output format. 0 - lowercase; 1 - uppercase */ + var hex_tab = hexcase ? "0123456789ABCDEF" : "0123456789abcdef"; + var str = ""; + for (var i = 0; i < binarray.length * 4; i++) { + str += hex_tab.charAt((binarray[i>>2] >> ((3 - i%4)*8+4)) & 0xF) + + hex_tab.charAt((binarray[i>>2] >> ((3 - i%4)*8 )) & 0xF); + } + return str; + } + function hex_sha256(s){ + return binb2hex(core_sha256(str2binb(s),s.length * chrsz)); + } + module.exports.hex_sha256 = hex_sha256; +}()); + +}, +"Common.js": function(module, exports, require){ +/* + * Copyright 2014 XWiki SAS + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +var PARANOIA = module.exports.PARANOIA = false; + +/* throw errors over non-compliant messages which would otherwise be treated as invalid */ +var TESTING = module.exports.TESTING = true; + +var assert = module.exports.assert = function (expr) { + if (!expr) { throw new Error("Failed assertion"); } +}; + +var isUint = module.exports.isUint = function (integer) { + return (typeof(integer) === 'number') && + (Math.floor(integer) === integer) && + (integer >= 0); +}; + +var randomASCII = module.exports.randomASCII = function (length) { + var content = []; + for (var i = 0; i < length; i++) { + content[i] = String.fromCharCode( Math.floor(Math.random()*256) % 57 + 65 ); + } + return content.join(''); +}; + +var strcmp = module.exports.strcmp = function (a, b) { + if (PARANOIA && typeof(a) !== 'string') { throw new Error(); } + if (PARANOIA && typeof(b) !== 'string') { throw new Error(); } + return ( (a === b) ? 0 : ( (a > b) ? 1 : -1 ) ); +} + +}, +"Message.js": function(module, exports, require){ +/* + * Copyright 2014 XWiki SAS + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +var Common = require('./Common'); +var Operation = require('./Operation'); +var Patch = require('./Patch'); +var Sha = require('./SHA256'); + +var Message = module.exports; + +var REGISTER = Message.REGISTER = 0; +var REGISTER_ACK = Message.REGISTER_ACK = 1; +var PATCH = Message.PATCH = 2; +var DISCONNECT = Message.DISCONNECT = 3; +var PING = Message.PING = 4; +var PONG = Message.PONG = 5; + +var check = Message.check = function(msg) { + Common.assert(msg.type === 'Message'); + Common.assert(typeof(msg.userName) === 'string'); + Common.assert(typeof(msg.authToken) === 'string'); + Common.assert(typeof(msg.channelId) === 'string'); + + if (msg.messageType === PATCH) { + Patch.check(msg.content); + Common.assert(typeof(msg.lastMsgHash) === 'string'); + } else if (msg.messageType === PING || msg.messageType === PONG) { + Common.assert(typeof(msg.lastMsgHash) === 'undefined'); + Common.assert(typeof(msg.content) === 'number'); + } else if (msg.messageType === REGISTER + || msg.messageType === REGISTER_ACK + || msg.messageType === DISCONNECT) + { + Common.assert(typeof(msg.lastMsgHash) === 'undefined'); + Common.assert(typeof(msg.content) === 'undefined'); + } else { + throw new Error("invalid message type [" + msg.messageType + "]"); + } +}; + +var create = Message.create = function (userName, authToken, channelId, type, content, lastMsgHash) { + var msg = { + type: 'Message', + userName: userName, + authToken: authToken, + channelId: channelId, + messageType: type, + content: content, + lastMsgHash: lastMsgHash + }; + if (Common.PARANOIA) { check(msg); } + return msg; +}; + +var toString = Message.toString = function (msg) { + if (Common.PARANOIA) { check(msg); } + var prefix = msg.messageType + ':'; + var content = ''; + if (msg.messageType === REGISTER) { + content = JSON.stringify([REGISTER]); + } else if (msg.messageType === PING || msg.messageType === PONG) { + content = JSON.stringify([msg.messageType, msg.content]); + } else if (msg.messageType === PATCH) { + content = JSON.stringify([PATCH, Patch.toObj(msg.content), msg.lastMsgHash]); + } + return msg.authToken.length + ":" + msg.authToken + + msg.userName.length + ":" + msg.userName + + msg.channelId.length + ":" + msg.channelId + + content.length + ':' + content; +}; + +var fromString = Message.fromString = function (str) { + var msg = str; + + var unameLen = msg.substring(0,msg.indexOf(':')); + msg = msg.substring(unameLen.length+1); + var userName = msg.substring(0,Number(unameLen)); + msg = msg.substring(userName.length); + + var channelIdLen = msg.substring(0,msg.indexOf(':')); + msg = msg.substring(channelIdLen.length+1); + var channelId = msg.substring(0,Number(channelIdLen)); + msg = msg.substring(channelId.length); + + var contentStrLen = msg.substring(0,msg.indexOf(':')); + msg = msg.substring(contentStrLen.length+1); + var contentStr = msg.substring(0,Number(contentStrLen)); + + Common.assert(contentStr.length === Number(contentStrLen)); + + var content = JSON.parse(contentStr); + var message; + if (content[0] === PATCH) { + message = create(userName, '', channelId, PATCH, Patch.fromObj(content[1]), content[2]); + } else if (content[0] === PING || content[0] === PONG) { + message = create(userName, '', channelId, content[0], content[1]); + } else { + message = create(userName, '', channelId, content[0]); + } + + // This check validates every operation in the patch. + check(message); + + return message +}; + +var hashOf = Message.hashOf = function (msg) { + if (Common.PARANOIA) { check(msg); } + var authToken = msg.authToken; + msg.authToken = ''; + var hash = Sha.hex_sha256(toString(msg)); + msg.authToken = authToken; + return hash; +}; + +}, +"ChainPad.js": function(module, exports, require){ +/* + * Copyright 2014 XWiki SAS + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +var Common = require('./Common'); +var Operation = module.exports.Operation = require('./Operation'); +var Patch = require('./Patch'); +var Message = require('./Message'); +var Sha = require('./SHA256'); + +var ChainPad = {}; + +// hex_sha256('') +var EMPTY_STR_HASH = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; +var ZERO = '0000000000000000000000000000000000000000000000000000000000000000'; + +var enterChainPad = function (realtime, func) { + return function () { + if (realtime.failed) { return; } + func.apply(null, arguments); + }; +}; + +var debug = function (realtime, msg) { + console.log("[" + realtime.userName + "] " + msg); +}; + +var schedule = function (realtime, func, timeout) { + if (!timeout) { + timeout = Math.floor(Math.random() * 2 * realtime.avgSyncTime); + } + var to = setTimeout(enterChainPad(realtime, function () { + realtime.schedules.splice(realtime.schedules.indexOf(to), 1); + func(); + }), timeout); + realtime.schedules.push(to); + return to; +}; + +var unschedule = function (realtime, schedule) { + var index = realtime.schedules.indexOf(schedule); + if (index > -1) { + realtime.schedules.splice(index, 1); + } + clearTimeout(schedule); +}; + +var onMessage = function (realtime, message, callback) { + if (!realtime.messageHandlers.length) { + callback("no onMessage() handler registered"); + } + for (var i = 0; i < realtime.messageHandlers.length; i++) { + realtime.messageHandlers[i](message, function () { + callback.apply(null, arguments); + callback = function () { }; + }); + } +}; + +var sync = function (realtime) { + if (Common.PARANOIA) { check(realtime); } + if (realtime.syncSchedule) { + unschedule(realtime, realtime.syncSchedule); + realtime.syncSchedule = null; + } else { + // we're currently waiting on something from the server. + return; + } + + realtime.uncommitted = Patch.simplify( + realtime.uncommitted, realtime.authDoc, realtime.config.operationSimplify); + + if (realtime.uncommitted.operations.length === 0) { + //debug(realtime, "No data to sync to the server, sleeping"); + realtime.syncSchedule = schedule(realtime, function () { sync(realtime); }); + return; + } + + var msg; + if (realtime.best === realtime.initialMessage) { + msg = realtime.initialMessage; + } else { + msg = Message.create(realtime.userName, + realtime.authToken, + realtime.channelId, + Message.PATCH, + realtime.uncommitted, + realtime.best.hashOf); + } + + var strMsg = Message.toString(msg); + + onMessage(realtime, strMsg, function (err) { + if (err) { + debug(realtime, "Posting to server failed [" + err + "]"); + } + }); + + var hash = Message.hashOf(msg); + + var timeout = schedule(realtime, function () { + debug(realtime, "Failed to send message ["+hash+"] to server"); + sync(realtime); + }, 10000 + (Math.random() * 5000)); + realtime.pending = { + hash: hash, + callback: function () { + if (realtime.initialMessage && realtime.initialMessage.hashOf === hash) { + debug(realtime, "initial Ack received ["+hash+"]"); + realtime.initialMessage = null; + } + unschedule(realtime, timeout); + realtime.syncSchedule = schedule(realtime, function () { sync(realtime); }, 0); + } + }; + if (Common.PARANOIA) { check(realtime); } +}; + +var getMessages = function (realtime) { + realtime.registered = true; + /*var to = schedule(realtime, function () { + throw new Error("failed to connect to the server"); + }, 5000);*/ + var msg = Message.create(realtime.userName, + realtime.authToken, + realtime.channelId, + Message.REGISTER); + onMessage(realtime, Message.toString(msg), function (err) { + if (err) { throw err; } + }); +}; + +var sendPing = function (realtime) { + realtime.pingSchedule = undefined; + realtime.lastPingTime = (new Date()).getTime(); + var msg = Message.create(realtime.userName, + realtime.authToken, + realtime.channelId, + Message.PING, + realtime.lastPingTime); + onMessage(realtime, Message.toString(msg), function (err) { + if (err) { throw err; } + }); +}; + +var onPong = function (realtime, msg) { + if (Common.PARANOIA) { + Common.assert(realtime.lastPingTime === Number(msg.content)); + } + realtime.lastPingLag = (new Date()).getTime() - Number(msg.content); + realtime.lastPingTime = 0; + realtime.pingSchedule = + schedule(realtime, function () { sendPing(realtime); }, realtime.pingCycle); +}; + +var create = ChainPad.create = function (userName, authToken, channelId, initialState, config) { + + var realtime = { + type: 'ChainPad', + + authDoc: '', + + config: config || {}, + + userName: userName, + authToken: authToken, + channelId: channelId, + + /** A patch representing all uncommitted work. */ + uncommitted: null, + + uncommittedDocLength: initialState.length, + + patchHandlers: [], + opHandlers: [], + + messageHandlers: [], + + schedules: [], + + syncSchedule: null, + + registered: false, + + avgSyncTime: 100, + + // this is only used if PARANOIA is enabled. + userInterfaceContent: undefined, + + failed: false, + + // hash and callback for previously send patch, currently in flight. + pending: null, + + messages: {}, + messagesByParent: {}, + + rootMessage: null, + + /** + * Set to the message which sets the initialState if applicable. + * Reset to null after the initial message has been successfully broadcasted. + */ + initialMessage: null, + + userListChangeHandlers: [], + userList: [], + + /** The schedule() for sending pings. */ + pingSchedule: undefined, + + lastPingLag: 0, + lastPingTime: 0, + + /** Average number of milliseconds between pings. */ + pingCycle: 5000 + }; + + if (Common.PARANOIA) { + realtime.userInterfaceContent = initialState; + } + + var zeroPatch = Patch.create(EMPTY_STR_HASH); + zeroPatch.inverseOf = Patch.invert(zeroPatch, ''); + zeroPatch.inverseOf.inverseOf = zeroPatch; + var zeroMsg = Message.create('', '', channelId, Message.PATCH, zeroPatch, ZERO); + zeroMsg.hashOf = Message.hashOf(zeroMsg); + zeroMsg.parentCount = 0; + realtime.messages[zeroMsg.hashOf] = zeroMsg; + (realtime.messagesByParent[zeroMsg.lastMessageHash] || []).push(zeroMsg); + realtime.rootMessage = zeroMsg; + realtime.best = zeroMsg; + + if (initialState === '') { + realtime.uncommitted = Patch.create(zeroPatch.inverseOf.parentHash); + return realtime; + } + + var initialOp = Operation.create(0, 0, initialState); + var initialStatePatch = Patch.create(zeroPatch.inverseOf.parentHash); + Patch.addOperation(initialStatePatch, initialOp); + initialStatePatch.inverseOf = Patch.invert(initialStatePatch, ''); + initialStatePatch.inverseOf.inverseOf = initialStatePatch; + + // flag this patch so it can be handled specially. + // Specifically, we never treat an initialStatePatch as our own, + // we let it be reverted to prevent duplication of data. + initialStatePatch.isInitialStatePatch = true; + initialStatePatch.inverseOf.isInitialStatePatch = true; + + realtime.authDoc = initialState; + if (Common.PARANOIA) { + realtime.userInterfaceContent = initialState; + } + initialMessage = Message.create(realtime.userName, + realtime.authToken, + realtime.channelId, + Message.PATCH, + initialStatePatch, + zeroMsg.hashOf); + initialMessage.hashOf = Message.hashOf(initialMessage); + initialMessage.parentCount = 1; + + realtime.messages[initialMessage.hashOf] = initialMessage; + (realtime.messagesByParent[initialMessage.lastMessageHash] || []).push(initialMessage); + + realtime.best = initialMessage; + realtime.uncommitted = Patch.create(initialStatePatch.inverseOf.parentHash); + realtime.initialMessage = initialMessage; + + return realtime; +}; + +var getParent = function (realtime, message) { + return message.parent = message.parent || realtime.messages[message.lastMsgHash]; +}; + +var check = ChainPad.check = function(realtime) { + Common.assert(realtime.type === 'ChainPad'); + Common.assert(typeof(realtime.authDoc) === 'string'); + + Patch.check(realtime.uncommitted, realtime.authDoc.length); + + var uiDoc = Patch.apply(realtime.uncommitted, realtime.authDoc); + if (uiDoc.length !== realtime.uncommittedDocLength) { + Common.assert(0); + } + if (realtime.userInterfaceContent !== '') { + Common.assert(uiDoc === realtime.userInterfaceContent); + } + + var doc = realtime.authDoc; + var patchMsg = realtime.best; + Common.assert(patchMsg.content.inverseOf.parentHash === realtime.uncommitted.parentHash); + var patches = []; + do { + patches.push(patchMsg); + doc = Patch.apply(patchMsg.content.inverseOf, doc); + } while ((patchMsg = getParent(realtime, patchMsg))); + Common.assert(doc === ''); + while ((patchMsg = patches.pop())) { + doc = Patch.apply(patchMsg.content, doc); + } + Common.assert(doc === realtime.authDoc); +}; + +var doOperation = ChainPad.doOperation = function (realtime, op) { + if (Common.PARANOIA) { + check(realtime); + realtime.userInterfaceContent = Operation.apply(op, realtime.userInterfaceContent); + } + Operation.check(op, realtime.uncommittedDocLength); + Patch.addOperation(realtime.uncommitted, op); + realtime.uncommittedDocLength += Operation.lengthChange(op); +}; + +var isAncestorOf = function (realtime, ancestor, decendent) { + if (!decendent || !ancestor) { return false; } + if (ancestor === decendent) { return true; } + return isAncestorOf(realtime, ancestor, getParent(realtime, decendent)); +}; + +var parentCount = function (realtime, message) { + if (typeof(message.parentCount) !== 'number') { + message.parentCount = parentCount(realtime, getParent(realtime, message)) + 1; + } + return message.parentCount; +}; + +var applyPatch = function (realtime, author, patch) { + if (author === realtime.userName && !patch.isInitialStatePatch) { + var inverseOldUncommitted = Patch.invert(realtime.uncommitted, realtime.authDoc); + var userInterfaceContent = Patch.apply(realtime.uncommitted, realtime.authDoc); + if (Common.PARANOIA) { + Common.assert(userInterfaceContent === realtime.userInterfaceContent); + } + realtime.uncommitted = Patch.merge(inverseOldUncommitted, patch); + realtime.uncommitted = Patch.invert(realtime.uncommitted, userInterfaceContent); + + } else { + + // don't bother trying to JSON.parse things if authDoc is not defined + // you'll never get past that point if you throw errors + if (realtime.authDoc) { + try { + var applied = Patch.apply(realtime.uncommitted, realtime.authDoc); + if (applied) { + JSON.parse(applied); + } else { + + console.error("[patchApplyError] Patch.apply(realtime.uncommitted, realtime.authDoc) // falsey"); + console.log({ + uncommitted: realtime.uncommitted, + authDoc: realtime.authdoc + }); + //throw new Error("[patchApplyError] Patch.apply(realtime.uncommitted, realtime.authDoc) // falsey"); + } + + } catch (err) { + console.error('[patchApplyError] realtime.uncommited => authDoc'); + console.error(err); + console.log({ + applied: applied, + uncommitted: realtime.uncommitted, + authDoc: realtime.authdoc + }); + + //throw new Error(); + } + try { + JSON.parse(realtime.authDoc); + } catch (err) { + console.error('[patchApplyError] realtime.authDoc'); + console.error(err); + //throw new Error(); + } + } + + realtime.uncommitted = + Patch.transform( + realtime.uncommitted, patch, realtime.authDoc, realtime.config.transformFunction); + } + realtime.uncommitted.parentHash = patch.inverseOf.parentHash; + + var temp = Patch.apply(patch, realtime.authDoc); + // changed by ansuz + /* + try { + JSON.parse(temp); + } catch (err) { + console.log(err); + throw new Error(); + }*/ + + realtime.authDoc = temp; + if (Common.PARANOIA) { + realtime.userInterfaceContent = Patch.apply(realtime.uncommitted, realtime.authDoc); + } +}; + +var revertPatch = function (realtime, author, patch) { + applyPatch(realtime, author, patch.inverseOf); +}; + +var getBestChild = function (realtime, msg) { + var best = msg; + (realtime.messagesByParent[msg.hashOf] || []).forEach(function (child) { + Common.assert(child.lastMsgHash === msg.hashOf); + child = getBestChild(realtime, child); + if (parentCount(realtime, child) > parentCount(realtime, best)) { best = child; } + }); + return best; +}; + +var userListChange = function (realtime) { + for (var i = 0; i < realtime.userListChangeHandlers.length; i++) { + var list = []; + list.push.apply(list, realtime.userList); + realtime.userListChangeHandlers[i](list); + } +}; + +var handleMessage = ChainPad.handleMessage = function (realtime, msgStr) { + + if (Common.PARANOIA) { check(realtime); } + var msg = Message.fromString(msgStr); + Common.assert(msg.channelId === realtime.channelId); + + if (msg.messageType === Message.REGISTER_ACK) { + debug(realtime, "registered"); + realtime.registered = true; + sendPing(realtime); + return; + } + + if (msg.messageType === Message.REGISTER) { + realtime.userList.push(msg.userName); + userListChange(realtime); + return; + } + + if (msg.messageType === Message.PONG) { + onPong(realtime, msg); + return; + } + + if (msg.messageType === Message.DISCONNECT) { + if (msg.userName === '') { + realtime.userList = []; + userListChange(realtime); + return; + } + var idx = realtime.userList.indexOf(msg.userName); + if (Common.PARANOIA) { Common.assert(idx > -1); } + if (idx > -1) { + realtime.userList.splice(idx, 1); + userListChange(realtime); + } + return; + } + + // otherwise it's a disconnect. + if (msg.messageType !== Message.PATCH) { return; } + + msg.hashOf = Message.hashOf(msg); + + if (realtime.pending && realtime.pending.hash === msg.hashOf) { + realtime.pending.callback(); + realtime.pending = null; + } + + if (realtime.messages[msg.hashOf]) { + debug(realtime, "Patch [" + msg.hashOf + "] is already known"); + if (Common.PARANOIA) { check(realtime); } + return; + } + + realtime.messages[msg.hashOf] = msg; + (realtime.messagesByParent[msg.lastMsgHash] = + realtime.messagesByParent[msg.lastMsgHash] || []).push(msg); + + if (!isAncestorOf(realtime, realtime.rootMessage, msg)) { + // we'll probably find the missing parent later. + debug(realtime, "Patch [" + msg.hashOf + "] not connected to root"); + if (Common.PARANOIA) { check(realtime); } + return; + } + + // of this message fills in a hole in the chain which makes another patch better, swap to the + // best child of this patch since longest chain always wins. + msg = getBestChild(realtime, msg); + var patch = msg.content; + + // Find the ancestor of this patch which is in the main chain, reverting as necessary + var toRevert = []; + var commonAncestor = realtime.best; + if (!isAncestorOf(realtime, realtime.best, msg)) { + var pcBest = parentCount(realtime, realtime.best); + var pcMsg = parentCount(realtime, msg); + if (pcBest < pcMsg + || (pcBest === pcMsg + && Common.strcmp(realtime.best.hashOf, msg.hashOf) > 0)) + { + // switch chains + while (commonAncestor && !isAncestorOf(realtime, commonAncestor, msg)) { + toRevert.push(commonAncestor); + commonAncestor = getParent(realtime, commonAncestor); + } + Common.assert(commonAncestor); + } else { + debug(realtime, "Patch [" + msg.hashOf + "] chain is ["+pcMsg+"] best chain is ["+pcBest+"]"); + if (Common.PARANOIA) { check(realtime); } + return; + } + } + + // Find the parents of this patch which are not in the main chain. + var toApply = []; + var current = msg; + do { + toApply.unshift(current); + current = getParent(realtime, current); + Common.assert(current); + } while (current !== commonAncestor); + + + var authDocAtTimeOfPatch = realtime.authDoc; + + for (var i = 0; i < toRevert.length; i++) { + authDocAtTimeOfPatch = Patch.apply(toRevert[i].content.inverseOf, authDocAtTimeOfPatch); + } + + // toApply.length-1 because we do not want to apply the new patch. + for (var i = 0; i < toApply.length-1; i++) { + if (typeof(toApply[i].content.inverseOf) === 'undefined') { + toApply[i].content.inverseOf = Patch.invert(toApply[i].content, authDocAtTimeOfPatch); + toApply[i].content.inverseOf.inverseOf = toApply[i].content; + } + authDocAtTimeOfPatch = Patch.apply(toApply[i].content, authDocAtTimeOfPatch); + } + + if (Sha.hex_sha256(authDocAtTimeOfPatch) !== patch.parentHash) { + debug(realtime, "patch [" + msg.hashOf + "] parentHash is not valid"); + if (Common.PARANOIA) { check(realtime); } + if (Common.TESTING) { throw new Error(); } + delete realtime.messages[msg.hashOf]; + return; + } + + var simplePatch = + Patch.simplify(patch, authDocAtTimeOfPatch, realtime.config.operationSimplify); + if (!Patch.equals(simplePatch, patch)) { + debug(realtime, "patch [" + msg.hashOf + "] can be simplified"); + if (Common.PARANOIA) { check(realtime); } + if (Common.TESTING) { throw new Error(); } + delete realtime.messages[msg.hashOf]; + return; + } + + patch.inverseOf = Patch.invert(patch, authDocAtTimeOfPatch); + patch.inverseOf.inverseOf = patch; + + realtime.uncommitted = Patch.simplify( + realtime.uncommitted, realtime.authDoc, realtime.config.operationSimplify); + var oldUserInterfaceContent = Patch.apply(realtime.uncommitted, realtime.authDoc); + if (Common.PARANOIA) { + Common.assert(oldUserInterfaceContent === realtime.userInterfaceContent); + } + + // Derive the patch for the user's uncommitted work + var uncommittedPatch = Patch.invert(realtime.uncommitted, realtime.authDoc); + + for (var i = 0; i < toRevert.length; i++) { + debug(realtime, "reverting [" + toRevert[i].hashOf + "]"); + uncommittedPatch = Patch.merge(uncommittedPatch, toRevert[i].content.inverseOf); + revertPatch(realtime, toRevert[i].userName, toRevert[i].content); + } + + for (var i = 0; i < toApply.length; i++) { + debug(realtime, "applying [" + toApply[i].hashOf + "]"); + uncommittedPatch = Patch.merge(uncommittedPatch, toApply[i].content); + applyPatch(realtime, toApply[i].userName, toApply[i].content); + } + + uncommittedPatch = Patch.merge(uncommittedPatch, realtime.uncommitted); + uncommittedPatch = Patch.simplify( + uncommittedPatch, oldUserInterfaceContent, realtime.config.operationSimplify); + + realtime.uncommittedDocLength += Patch.lengthChange(uncommittedPatch); + realtime.best = msg; + + if (Common.PARANOIA) { + // apply the uncommittedPatch to the userInterface content. + var newUserInterfaceContent = Patch.apply(uncommittedPatch, oldUserInterfaceContent); + Common.assert(realtime.userInterfaceContent.length === realtime.uncommittedDocLength); + Common.assert(newUserInterfaceContent === realtime.userInterfaceContent); + } + + if (uncommittedPatch.operations.length) { + // push the uncommittedPatch out to the user interface. + for (var i = 0; i < realtime.patchHandlers.length; i++) { + realtime.patchHandlers[i](uncommittedPatch); + } + if (realtime.opHandlers.length) { + for (var i = uncommittedPatch.operations.length-1; i >= 0; i--) { + for (var j = 0; j < realtime.opHandlers.length; j++) { + realtime.opHandlers[j](uncommittedPatch.operations[i]); + } + } + } + } + if (Common.PARANOIA) { check(realtime); } +}; + +var wasEverState = function (content, realtime) { + Common.assert(typeof(content) === 'string'); + // without this we would never get true on the ^HEAD + if (realtime.authDoc === content) { + return true; + } + + var hash = Sha.hex_sha256(content); + + var patchMsg = realtime.best; + do { + if (patchMsg.content.parentHash === hash) { return true; } + } while ((patchMsg = getParent(realtime, patchMsg))); + return false; +}; + +var getDepthOfState = function (content, minDepth, realtime) { + Common.assert(typeof(content) === 'string'); + + // minimum depth is an optional argument which defaults to zero + var minDepth = minDepth || 0; + + if (minDepth === 0 && realtime.authDoc === content) { + return 0; + } + + var hash = Sha.hex_sha256(content); + + var patchMsg = realtime.best; + var depth = 0; + + do { + if (depth < minDepth) { + // you haven't exceeded the minimum depth + } else { + //console.log("Exceeded minimum depth"); + // you *have* exceeded the minimum depth + if (patchMsg.content.parentHash === hash) { + // you found it! + return depth + 1; + } + } + depth++; + } while ((patchMsg = getParent(realtime, patchMsg))); + return; +}; + +module.exports.create = function (userName, authToken, channelId, initialState, conf) { + Common.assert(typeof(userName) === 'string'); + Common.assert(typeof(authToken) === 'string'); + Common.assert(typeof(channelId) === 'string'); + Common.assert(typeof(initialState) === 'string'); + var realtime = ChainPad.create(userName, authToken, channelId, initialState, conf); + return { + onPatch: enterChainPad(realtime, function (handler) { + Common.assert(typeof(handler) === 'function'); + realtime.patchHandlers.push(handler); + }), + onRemove: enterChainPad(realtime, function (handler) { + Common.assert(typeof(handler) === 'function'); + realtime.opHandlers.unshift(function (op) { + if (op.toRemove > 0) { handler(op.offset, op.toRemove); } + }); + }), + onInsert: enterChainPad(realtime, function (handler) { + Common.assert(typeof(handler) === 'function'); + realtime.opHandlers.push(function (op) { + if (op.toInsert.length > 0) { handler(op.offset, op.toInsert); } + }); + }), + remove: enterChainPad(realtime, function (offset, numChars) { + doOperation(realtime, Operation.create(offset, numChars, '')); + }), + insert: enterChainPad(realtime, function (offset, str) { + doOperation(realtime, Operation.create(offset, 0, str)); + }), + onMessage: enterChainPad(realtime, function (handler) { + Common.assert(typeof(handler) === 'function'); + realtime.messageHandlers.push(handler); + }), + message: enterChainPad(realtime, function (message) { + handleMessage(realtime, message); + }), + start: enterChainPad(realtime, function () { + getMessages(realtime); + if (realtime.syncSchedule) { unschedule(realtime, realtime.syncSchedule); } + realtime.syncSchedule = schedule(realtime, function () { sync(realtime); }); + }), + abort: enterChainPad(realtime, function () { + realtime.schedules.forEach(function (s) { clearTimeout(s) }); + }), + sync: enterChainPad(realtime, function () { + sync(realtime); + }), + getAuthDoc: function () { return realtime.authDoc; }, + getUserDoc: function () { return Patch.apply(realtime.uncommitted, realtime.authDoc); }, + onUserListChange: enterChainPad(realtime, function (handler) { + Common.assert(typeof(handler) === 'function'); + realtime.userListChangeHandlers.push(handler); + }), + getLag: function () { + if (realtime.lastPingTime) { + return { waiting:1, lag: (new Date()).getTime() - realtime.lastPingTime }; + } + return { waiting:0, lag: realtime.lastPingLag }; + }, + wasEverState: function (content) { + return wasEverState(content, realtime); + }, + getDepthOfState: function (content, minDepth) { + return getDepthOfState(content, minDepth, realtime); + } + }; +}; + +}, +"Operation.js": function(module, exports, require){ +/* + * Copyright 2014 XWiki SAS + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +var Common = require('./Common'); + +var Operation = module.exports; + +var check = Operation.check = function (op, docLength_opt) { + Common.assert(op.type === 'Operation'); + Common.assert(Common.isUint(op.offset)); + Common.assert(Common.isUint(op.toRemove)); + Common.assert(typeof(op.toInsert) === 'string'); + Common.assert(op.toRemove > 0 || op.toInsert.length > 0); + Common.assert(typeof(docLength_opt) !== 'number' || op.offset + op.toRemove <= docLength_opt); +}; + +var create = Operation.create = function (offset, toRemove, toInsert) { + var out = { + type: 'Operation', + offset: offset || 0, + toRemove: toRemove || 0, + toInsert: toInsert || '', + }; + if (Common.PARANOIA) { check(out); } + return out; +}; + +var toObj = Operation.toObj = function (op) { + if (Common.PARANOIA) { check(op); } + return [op.offset,op.toRemove,op.toInsert]; +}; + +var fromObj = Operation.fromObj = function (obj) { + Common.assert(Array.isArray(obj) && obj.length === 3); + return create(obj[0], obj[1], obj[2]); +}; + +var clone = Operation.clone = function (op) { + return create(op.offset, op.toRemove, op.toInsert); +}; + +/** + * @param op the operation to apply. + * @param doc the content to apply the operation on + */ +var apply = Operation.apply = function (op, doc) +{ + if (Common.PARANOIA) { + check(op); + Common.assert(typeof(doc) === 'string'); + Common.assert(op.offset + op.toRemove <= doc.length); + } + return doc.substring(0,op.offset) + op.toInsert + doc.substring(op.offset + op.toRemove); +}; + +var invert = Operation.invert = function (op, doc) { + if (Common.PARANOIA) { + check(op); + Common.assert(typeof(doc) === 'string'); + Common.assert(op.offset + op.toRemove <= doc.length); + } + var rop = clone(op); + rop.toInsert = doc.substring(op.offset, op.offset + op.toRemove); + rop.toRemove = op.toInsert.length; + return rop; +}; + +var simplify = Operation.simplify = function (op, doc) { + if (Common.PARANOIA) { + check(op); + Common.assert(typeof(doc) === 'string'); + Common.assert(op.offset + op.toRemove <= doc.length); + } + var rop = invert(op, doc); + op = clone(op); + + var minLen = Math.min(op.toInsert.length, rop.toInsert.length); + var i; + for (i = 0; i < minLen && rop.toInsert[i] === op.toInsert[i]; i++) ; + op.offset += i; + op.toRemove -= i; + op.toInsert = op.toInsert.substring(i); + rop.toInsert = rop.toInsert.substring(i); + + if (rop.toInsert.length === op.toInsert.length) { + for (i = rop.toInsert.length-1; i >= 0 && rop.toInsert[i] === op.toInsert[i]; i--) ; + op.toInsert = op.toInsert.substring(0, i+1); + op.toRemove = i+1; + } + + if (op.toRemove === 0 && op.toInsert.length === 0) { return null; } + return op; +}; + +var equals = Operation.equals = function (opA, opB) { + return (opA.toRemove === opB.toRemove + && opA.toInsert === opB.toInsert + && opA.offset === opB.offset); +}; + +var lengthChange = Operation.lengthChange = function (op) +{ + if (Common.PARANOIA) { check(op); } + return op.toInsert.length - op.toRemove; +}; + +/* + * @return the merged operation OR null if the result of the merger is a noop. + */ +var merge = Operation.merge = function (oldOpOrig, newOpOrig) { + if (Common.PARANOIA) { + check(newOpOrig); + check(oldOpOrig); + } + + var newOp = clone(newOpOrig); + var oldOp = clone(oldOpOrig); + var offsetDiff = newOp.offset - oldOp.offset; + + if (newOp.toRemove > 0) { + var origOldInsert = oldOp.toInsert; + oldOp.toInsert = ( + oldOp.toInsert.substring(0,offsetDiff) + + oldOp.toInsert.substring(offsetDiff + newOp.toRemove) + ); + newOp.toRemove -= (origOldInsert.length - oldOp.toInsert.length); + if (newOp.toRemove < 0) { newOp.toRemove = 0; } + + oldOp.toRemove += newOp.toRemove; + newOp.toRemove = 0; + } + + if (offsetDiff < 0) { + oldOp.offset += offsetDiff; + oldOp.toInsert = newOp.toInsert + oldOp.toInsert; + + } else if (oldOp.toInsert.length === offsetDiff) { + oldOp.toInsert = oldOp.toInsert + newOp.toInsert; + + } else if (oldOp.toInsert.length > offsetDiff) { + oldOp.toInsert = ( + oldOp.toInsert.substring(0,offsetDiff) + + newOp.toInsert + + oldOp.toInsert.substring(offsetDiff) + ); + } else { + throw new Error("should never happen\n" + + JSON.stringify([oldOpOrig,newOpOrig], null, ' ')); + } + + if (oldOp.toInsert === '' && oldOp.toRemove === 0) { + return null; + } + if (Common.PARANOIA) { check(oldOp); } + + return oldOp; +}; + +/** + * If the new operation deletes what the old op inserted or inserts content in the middle of + * the old op's content or if they abbut one another, they should be merged. + */ +var shouldMerge = Operation.shouldMerge = function (oldOp, newOp) { + if (Common.PARANOIA) { + check(oldOp); + check(newOp); + } + if (newOp.offset < oldOp.offset) { + return (oldOp.offset <= (newOp.offset + newOp.toRemove)); + } else { + return (newOp.offset <= (oldOp.offset + oldOp.toInsert.length)); + } +}; + +/** + * Rebase newOp against oldOp. + * + * @param oldOp the eariler operation to have happened. + * @param newOp the later operation to have happened (in time). + * @return either the untouched newOp if it need not be rebased, + * the rebased clone of newOp if it needs rebasing, or + * null if newOp and oldOp must be merged. + */ +var rebase = Operation.rebase = function (oldOp, newOp) { + if (Common.PARANOIA) { + check(oldOp); + check(newOp); + } + if (newOp.offset < oldOp.offset) { return newOp; } + newOp = clone(newOp); + newOp.offset += oldOp.toRemove; + newOp.offset -= oldOp.toInsert.length; + return newOp; +}; + +/** + * this is a lossy and dirty algorithm, everything else is nice but transformation + * has to be lossy because both operations have the same base and they diverge. + * This could be made nicer and/or tailored to a specific data type. + * + * @param toTransform the operation which is converted *MUTATED*. + * @param transformBy an existing operation which also has the same base. + * @return toTransform *or* null if the result is a no-op. + */ + +var transform0 = Operation.transform0 = function (text, toTransformOrig, transformByOrig) { + // Cloning the original transformations makes this algorithm such that it + // **DOES NOT MUTATE ANYMORE** + var toTransform = Operation.clone(toTransformOrig); + var transformBy = Operation.clone(transformByOrig); + + if (toTransform.offset > transformBy.offset) { + if (toTransform.offset > transformBy.offset + transformBy.toRemove) { + // simple rebase + toTransform.offset -= transformBy.toRemove; + toTransform.offset += transformBy.toInsert.length; + return toTransform; + } + // goto the end, anything you deleted that they also deleted should be skipped. + var newOffset = transformBy.offset + transformBy.toInsert.length; + toTransform.toRemove = 0; //-= (newOffset - toTransform.offset); + if (toTransform.toRemove < 0) { toTransform.toRemove = 0; } + toTransform.offset = newOffset; + if (toTransform.toInsert.length === 0 && toTransform.toRemove === 0) { + return null; + } + return toTransform; + } + if (toTransform.offset + toTransform.toRemove < transformBy.offset) { + return toTransform; + } + toTransform.toRemove = transformBy.offset - toTransform.offset; + if (toTransform.toInsert.length === 0 && toTransform.toRemove === 0) { + return null; + } + return toTransform; +}; + +/** + * @param toTransform the operation which is converted + * @param transformBy an existing operation which also has the same base. + * @return a modified clone of toTransform *or* toTransform itself if no change was made. + */ +var transform = Operation.transform = function (text, toTransform, transformBy, transformFunction) { + if (Common.PARANOIA) { + check(toTransform); + check(transformBy); + } + transformFunction = transformFunction || transform0; + toTransform = clone(toTransform); + var result = transformFunction(text, toTransform, transformBy); + if (Common.PARANOIA && result) { check(result); } + return result; +}; + +/** Used for testing. */ +var random = Operation.random = function (docLength) { + Common.assert(Common.isUint(docLength)); + var offset = Math.floor(Math.random() * 100000000 % docLength) || 0; + var toRemove = Math.floor(Math.random() * 100000000 % (docLength - offset)) || 0; + var toInsert = ''; + do { + var toInsert = Common.randomASCII(Math.floor(Math.random() * 20)); + } while (toRemove === 0 && toInsert === ''); + return create(offset, toRemove, toInsert); +}; + +} +}; +ChainPad = r("ChainPad.js");}()); diff --git a/www/_socket/realtime-input.js b/www/_socket/realtime-input.js index 565200aba..c592d49f9 100644 --- a/www/_socket/realtime-input.js +++ b/www/_socket/realtime-input.js @@ -20,7 +20,7 @@ define([ '/common/crypto.js', '/_socket/toolbar.js', '/_socket/text-patcher.js', - '/common/chainpad.js', + '/_socket/chainpad.js', '/bower_components/jquery/dist/jquery.min.js', ], function (Messages,/*FIXME*/ ReconnectingWebSocket, Crypto, Toolbar, TextPatcher) { var $ = window.jQuery; From b59a14c5ac6e11de3b991f2c41585a112ca3246d Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 1 Apr 2016 11:20:19 +0200 Subject: [PATCH 39/69] merge hyperjson changes from realtime xwiki more resilient class serialization. comments --- www/common/hyperjson.js | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/www/common/hyperjson.js b/www/common/hyperjson.js index 71ba71987..ff04a0da9 100644 --- a/www/common/hyperjson.js +++ b/www/common/hyperjson.js @@ -1,5 +1,4 @@ define([], function () { - // this makes recursing a lot simpler var isArray = function (A) { return Object.prototype.toString.call(A)==='[object Array]'; @@ -39,8 +38,14 @@ define([], function () { return cb(hj[0], hj[1], children); }; - var prependDot = function (token) { - return '.' + token; + var classify = function (token) { + return '.' + token.trim(); + }; + + var isValidClass = function (x) { + if (x && /\S/.test(x)) { + return true; + } }; var isTruthy = function (x) { @@ -54,13 +59,13 @@ define([], function () { if(!el.attributes){ return; } - if (predicate) { if (!predicate(el)) { // shortcircuit return; } } + var attributes = {}; var i = 0; @@ -84,19 +89,25 @@ define([], function () { var sel = el.tagName; if(attributes.id){ + // we don't have to do much to validate IDs because the browser + // will only permit one id to exist + // unless we come across a strange browser in the wild sel = sel +'#'+ attributes.id; delete attributes.id; } if(attributes.class){ + // actually parse out classes so that we produce a valid selector + // string. leading or trailing spaces would have caused it to choke + // these are really common in generated html /* TODO this can be done with RegExps alone, and it will be faster but this works and is a little less error prone, albeit slower come back and speed it up when it comes time to optimize */ - sel = sel + attributes.class - .split(/\s+/) - .filter(isTruthy) - .map(prependDot) - .join(''); + .split(/\s+/g) + .filter(isValidClass) + .map(classify) + .join('') + .replace(/\.\./g, '.'); delete attributes.class; } result.push(sel); @@ -109,11 +120,9 @@ define([], function () { // js hint complains if we use 'var' here i = 0; - for(; i < el.childNodes.length; i++){ children.push(DOM2HyperJSON(el.childNodes[i], predicate, filter)); } - result.push(children.filter(isTruthy)); if (filter) { From 842b9d4243ca7784a7f4d9fe4170092d340877f4 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 7 Apr 2016 15:48:38 +0200 Subject: [PATCH 40/69] Generalize _socket/realtime-input.js a little more Anything JSON related should be at the application layer --- www/_socket/realtime-input.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/www/_socket/realtime-input.js b/www/_socket/realtime-input.js index c592d49f9..24e2e140f 100644 --- a/www/_socket/realtime-input.js +++ b/www/_socket/realtime-input.js @@ -147,8 +147,9 @@ define([ passwd, // password, to be deprecated (maybe) channel, // the channel we're to connect to - // initialState argument. (optional) - config.initialState || '{}', + /* optional unless your application expects JSON + from getUserDoc */ + config.initialState || '', // transform function (optional), which handles conflicts { transformFunction: config.transformFunction }); @@ -160,7 +161,6 @@ define([ // this is a problem warn("realtime.getUserDoc() !== newText"); } - //try{throw new Error();}catch(e){console.log(e.stack);} }; // pass your shiny new realtime into initialization functions From f5b0e1a5df7c51be6615885b3a51db8cec8fcecd Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 7 Apr 2016 15:50:56 +0200 Subject: [PATCH 41/69] Add comments, debug variables. initialize better Expose Hyperscript via window.REALTIME_MODULE Make sure to pass in a JSON.parse'able initial state. Add comments detailing the problems with not using setAttribute --- www/_socket/main.js | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index 0974a66f0..3b2095be4 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -28,7 +28,9 @@ define([ toolbar; var module = window.REALTIME_MODULE = { - localChangeInProgress: 0 + localChangeInProgress: 0, + Hyperjson: Hyperjson, + Hyperscript: Hyperscript }; var isNotMagicLine = function (el) { @@ -163,7 +165,8 @@ define([ doc: inner, // provide initialstate... - initialState: JSON.stringify(Hyperjson.fromDOM(inner, isNotMagicLine)), + initialState: JSON.stringify(Hyperjson + .fromDOM(inner, isNotMagicLine)) || '{}', // really basic operational transform // reject patch if it results in invalid JSON @@ -200,6 +203,21 @@ define([ var userDocStateDom = hjsonToDom(JSON.parse(shjson)); localWorkInProgress(2); // check again + + + /* in the DOM contentEditable is "false" + while "contenteditable" is undefined. + + When it goes over the wire, it seems hyperjson transforms it. + of course, hyperjson simply gets attributes from the DOM. + + el.attributes returns 'contenteditable', so we have to correct for that + + There are quite possibly all sorts of other attributes which might lose + information, and we won't know what they are until after we've lost them. + + this comes from hyperscript line 101. FIXME maybe + */ userDocStateDom.setAttribute("contenteditable", "true"); // lol wtf localWorkInProgress(3); // check again var patch = (DD).diff(inner, userDocStateDom); From ec64e0d3816a9999b7f6d330e1f114b9411b40f0 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 7 Apr 2016 15:55:57 +0200 Subject: [PATCH 42/69] use getAttribute to inspect document elements el.getAttribute('attr') is more reliable than el[attr]. --- www/_socket/main.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index 3b2095be4..e943aa949 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -36,7 +36,8 @@ define([ var isNotMagicLine = function (el) { // factor as: // return !(el.tagName === 'SPAN' && el.contentEditable === 'false'); - var filter = (el.tagName === 'SPAN' && el.contentEditable === 'false'); + var filter = (el.tagName === 'SPAN' && + el.getAttribute('contentEditable') === 'false'); if (filter) { console.log("[hyperjson.serializer] prevented an element" + "from being serialized:", el); @@ -97,7 +98,7 @@ define([ we should check when such an element is going to be removed, and prevent that from happening. */ if (info.node && info.node.tagName === 'SPAN' && - info.node.contentEditable === "true") { + info.node.getAttribute('contentEditable') === "false") { // it seems to be a magicline plugin element... if (info.diff.action === 'removeElement') { // and you're about to remove it... From cd357a9136ca31f0f9448c7591d798abc08fa4d2 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 7 Apr 2016 17:03:30 +0200 Subject: [PATCH 43/69] turn an alert into a console.error --- www/_socket/main.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index e943aa949..f334929e9 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -190,8 +190,7 @@ define([ var localWorkInProgress = function (stage) { if (module.localChangeInProgress) { console.error("Applied a change while a local patch was in progress"); - alert("local work was interrupted at stage: " + stage); - //module.realtimeInput.onLocal(); + console.error("local work was interrupted at stage: " + stage); return true; } return false; From 05108efdfacebb1ee291b650539ee683e5a2aeeb Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 09:44:08 +0200 Subject: [PATCH 44/69] correct chainpad's transformation function addresses RTCHAINPAD-3 Pending further testing, this change will still need to be corrected in https://github.com/xwiki-contrib/chainpad/ --- www/common/chainpad.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/www/common/chainpad.js b/www/common/chainpad.js index 502e502ab..071c595e8 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -223,7 +223,10 @@ var transform = Patch.transform = function (origToTransform, transformBy, doc, t toTransform = clone(origToTransform); var text = doc; for (var i = toTransform.operations.length-1; i >= 0; i--) { - text = Operation.apply(toTransform.operations[i], text); + /* this line caused diffs to be applied against the incorrect text + which resulted in bad merges that weren't noticeable until we + tried to patch json. */ + //text = Operation.apply(toTransform.operations[i], text); for (var j = transformBy.operations.length-1; j >= 0; j--) { toTransform.operations[i] = Operation.transform(text, toTransform.operations[i], From 4ba68eb2bfe24972b626ff85d1a4facbdab18d9c Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 09:52:36 +0200 Subject: [PATCH 45/69] fix leaky variable in chainpad Patch.transform declared a 'toTransform' variable without using 'var' this caused it to leak onto the window during operational transformations --- www/common/chainpad.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/common/chainpad.js b/www/common/chainpad.js index 071c595e8..ba8a4d8ca 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -220,7 +220,7 @@ var transform = Patch.transform = function (origToTransform, transformBy, doc, t Common.assert(origToTransform.parentHash === transformBy.parentHash); var resultOfTransformBy = apply(transformBy, doc); - toTransform = clone(origToTransform); + var toTransform = clone(origToTransform); var text = doc; for (var i = toTransform.operations.length-1; i >= 0; i--) { /* this line caused diffs to be applied against the incorrect text From 4071a3aa25f3767511e0ba2a0c9e839a2b09d46d Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 09:55:50 +0200 Subject: [PATCH 46/69] revert to using the chainpad in /common/ now that we have found the problem that caused the backspace bug we don't need a special chainpad for testing. --- www/_socket/chainpad.js | 1558 --------------------------------- www/_socket/realtime-input.js | 2 +- 2 files changed, 1 insertion(+), 1559 deletions(-) delete mode 100644 www/_socket/chainpad.js diff --git a/www/_socket/chainpad.js b/www/_socket/chainpad.js deleted file mode 100644 index ead48ee26..000000000 --- a/www/_socket/chainpad.js +++ /dev/null @@ -1,1558 +0,0 @@ -(function(){ -var r=function(){var e="function"==typeof require&&require,r=function(i,o,u){o||(o=0);var n=r.resolve(i,o),t=r.m[o][n];if(!t&&e){if(t=e(n))return t}else if(t&&t.c&&(o=t.c,n=t.m,t=r.m[o][t.m],!t))throw new Error('failed to require "'+n+'" from '+o);if(!t)throw new Error('failed to require "'+i+'" from '+u);return t.exports||(t.exports={},t.call(t.exports,t,t.exports,r.relative(n,o))),t.exports};return r.resolve=function(e,n){var i=e,t=e+".js",o=e+"/index.js";return r.m[n][t]&&t?t:r.m[n][o]&&o?o:i},r.relative=function(e,t){return function(n){if("."!=n.charAt(0))return r(n,t,e);var o=e.split("/"),f=n.split("/");o.pop();for(var i=0;i. - */ -var Common = require('./Common'); -var Operation = require('./Operation'); -var Sha = require('./SHA256'); - -var Patch = module.exports; - -var create = Patch.create = function (parentHash) { - return { - type: 'Patch', - operations: [], - parentHash: parentHash - }; -}; - -var check = Patch.check = function (patch, docLength_opt) { - Common.assert(patch.type === 'Patch'); - Common.assert(Array.isArray(patch.operations)); - Common.assert(/^[0-9a-f]{64}$/.test(patch.parentHash)); - for (var i = patch.operations.length - 1; i >= 0; i--) { - Operation.check(patch.operations[i], docLength_opt); - if (i > 0) { - Common.assert(!Operation.shouldMerge(patch.operations[i], patch.operations[i-1])); - } - if (typeof(docLength_opt) === 'number') { - docLength_opt += Operation.lengthChange(patch.operations[i]); - } - } -}; - -var toObj = Patch.toObj = function (patch) { - if (Common.PARANOIA) { check(patch); } - var out = new Array(patch.operations.length+1); - var i; - for (i = 0; i < patch.operations.length; i++) { - out[i] = Operation.toObj(patch.operations[i]); - } - out[i] = patch.parentHash; - return out; -}; - -var fromObj = Patch.fromObj = function (obj) { - Common.assert(Array.isArray(obj) && obj.length > 0); - var patch = create(); - var i; - for (i = 0; i < obj.length-1; i++) { - patch.operations[i] = Operation.fromObj(obj[i]); - } - patch.parentHash = obj[i]; - if (Common.PARANOIA) { check(patch); } - return patch; -}; - -var hash = function (text) { - return Sha.hex_sha256(text); -}; - -var addOperation = Patch.addOperation = function (patch, op) { - if (Common.PARANOIA) { - check(patch); - Operation.check(op); - } - for (var i = 0; i < patch.operations.length; i++) { - if (Operation.shouldMerge(patch.operations[i], op)) { - op = Operation.merge(patch.operations[i], op); - patch.operations.splice(i,1); - if (op === null) { - //console.log("operations cancelled eachother"); - return; - } - i--; - } else { - var out = Operation.rebase(patch.operations[i], op); - if (out === op) { - // op could not be rebased further, insert it here to keep the list ordered. - patch.operations.splice(i,0,op); - return; - } else { - op = out; - // op was rebased, try rebasing it against the next operation. - } - } - } - patch.operations.push(op); - if (Common.PARANOIA) { check(patch); } -}; - -var clone = Patch.clone = function (patch) { - if (Common.PARANOIA) { check(patch); } - var out = create(); - out.parentHash = patch.parentHash; - for (var i = 0; i < patch.operations.length; i++) { - out.operations[i] = Operation.clone(patch.operations[i]); - } - return out; -}; - -var merge = Patch.merge = function (oldPatch, newPatch) { - if (Common.PARANOIA) { - check(oldPatch); - check(newPatch); - } - oldPatch = clone(oldPatch); - for (var i = newPatch.operations.length-1; i >= 0; i--) { - addOperation(oldPatch, newPatch.operations[i]); - } - return oldPatch; -}; - -var apply = Patch.apply = function (patch, doc) -{ - if (Common.PARANOIA) { - check(patch); - Common.assert(typeof(doc) === 'string'); - Common.assert(Sha.hex_sha256(doc) === patch.parentHash); - } - var newDoc = doc; - for (var i = patch.operations.length-1; i >= 0; i--) { - newDoc = Operation.apply(patch.operations[i], newDoc); - } - return newDoc; -}; - -var lengthChange = Patch.lengthChange = function (patch) -{ - if (Common.PARANOIA) { check(patch); } - var out = 0; - for (var i = 0; i < patch.operations.length; i++) { - out += Operation.lengthChange(patch.operations[i]); - } - return out; -}; - -var invert = Patch.invert = function (patch, doc) -{ - if (Common.PARANOIA) { - check(patch); - Common.assert(typeof(doc) === 'string'); - Common.assert(Sha.hex_sha256(doc) === patch.parentHash); - } - var rpatch = create(); - var newDoc = doc; - for (var i = patch.operations.length-1; i >= 0; i--) { - rpatch.operations[i] = Operation.invert(patch.operations[i], newDoc); - newDoc = Operation.apply(patch.operations[i], newDoc); - } - for (var i = rpatch.operations.length-1; i >= 0; i--) { - for (var j = i - 1; j >= 0; j--) { - rpatch.operations[i].offset += rpatch.operations[j].toRemove; - rpatch.operations[i].offset -= rpatch.operations[j].toInsert.length; - } - } - rpatch.parentHash = Sha.hex_sha256(newDoc); - if (Common.PARANOIA) { check(rpatch); } - return rpatch; -}; - -var simplify = Patch.simplify = function (patch, doc, operationSimplify) -{ - if (Common.PARANOIA) { - check(patch); - Common.assert(typeof(doc) === 'string'); - Common.assert(Sha.hex_sha256(doc) === patch.parentHash); - } - operationSimplify = operationSimplify || Operation.simplify; - var spatch = create(patch.parentHash); - var newDoc = doc; - var outOps = []; - var j = 0; - for (var i = patch.operations.length-1; i >= 0; i--) { - outOps[j] = operationSimplify(patch.operations[i], newDoc, Operation.simplify); - if (outOps[j]) { - newDoc = Operation.apply(outOps[j], newDoc); - j++; - } - } - spatch.operations = outOps.reverse(); - if (!spatch.operations[0]) { - spatch.operations.shift(); - } - if (Common.PARANOIA) { - check(spatch); - } - return spatch; -}; - -var equals = Patch.equals = function (patchA, patchB) { - if (patchA.operations.length !== patchB.operations.length) { return false; } - for (var i = 0; i < patchA.operations.length; i++) { - if (!Operation.equals(patchA.operations[i], patchB.operations[i])) { return false; } - } - return true; -}; - -var transform = Patch.transform = function (origToTransform, transformBy, doc, transformFunction) { - if (Common.PARANOIA) { - check(origToTransform, doc.length); - check(transformBy, doc.length); - Common.assert(Sha.hex_sha256(doc) === origToTransform.parentHash); - } - Common.assert(origToTransform.parentHash === transformBy.parentHash); - var resultOfTransformBy = apply(transformBy, doc); - - toTransform = clone(origToTransform); - var text = doc; - for (var i = toTransform.operations.length-1; i >= 0; i--) { - text = Operation.apply(toTransform.operations[i], text); - for (var j = transformBy.operations.length-1; j >= 0; j--) { - toTransform.operations[i] = Operation.transform(text, - toTransform.operations[i], - transformBy.operations[j], - transformFunction); - if (!toTransform.operations[i]) { - break; - } - } - if (Common.PARANOIA && toTransform.operations[i]) { - Operation.check(toTransform.operations[i], resultOfTransformBy.length); - } - } - var out = create(transformBy.parentHash); - for (var i = toTransform.operations.length-1; i >= 0; i--) { - if (toTransform.operations[i]) { - addOperation(out, toTransform.operations[i]); - } - } - - out.parentHash = Sha.hex_sha256(resultOfTransformBy); - - if (Common.PARANOIA) { - check(out, resultOfTransformBy.length); - } - return out; -}; - -var random = Patch.random = function (doc, opCount) { - Common.assert(typeof(doc) === 'string'); - opCount = opCount || (Math.floor(Math.random() * 30) + 1); - var patch = create(Sha.hex_sha256(doc)); - var docLength = doc.length; - while (opCount-- > 0) { - var op = Operation.random(docLength); - docLength += Operation.lengthChange(op); - addOperation(patch, op); - } - check(patch); - return patch; -}; - -}, -"SHA256.js": function(module, exports, require){ -/* A JavaScript implementation of the Secure Hash Algorithm, SHA-256 - * Version 0.3 Copyright Angel Marin 2003-2004 - http://anmar.eu.org/ - * Distributed under the BSD License - * Some bits taken from Paul Johnston's SHA-1 implementation - */ -(function () { - var chrsz = 8; /* bits per input character. 8 - ASCII; 16 - Unicode */ - function safe_add (x, y) { - var lsw = (x & 0xFFFF) + (y & 0xFFFF); - var msw = (x >> 16) + (y >> 16) + (lsw >> 16); - return (msw << 16) | (lsw & 0xFFFF); - } - function S (X, n) {return ( X >>> n ) | (X << (32 - n));} - function R (X, n) {return ( X >>> n );} - function Ch(x, y, z) {return ((x & y) ^ ((~x) & z));} - function Maj(x, y, z) {return ((x & y) ^ (x & z) ^ (y & z));} - function Sigma0256(x) {return (S(x, 2) ^ S(x, 13) ^ S(x, 22));} - function Sigma1256(x) {return (S(x, 6) ^ S(x, 11) ^ S(x, 25));} - function Gamma0256(x) {return (S(x, 7) ^ S(x, 18) ^ R(x, 3));} - function Gamma1256(x) {return (S(x, 17) ^ S(x, 19) ^ R(x, 10));} - function newArray (n) { - var a = []; - for (;n>0;n--) { - a.push(undefined); - } - return a; - } - function core_sha256 (m, l) { - var K = [0x428A2F98,0x71374491,0xB5C0FBCF,0xE9B5DBA5,0x3956C25B,0x59F111F1,0x923F82A4,0xAB1C5ED5,0xD807AA98,0x12835B01,0x243185BE,0x550C7DC3,0x72BE5D74,0x80DEB1FE,0x9BDC06A7,0xC19BF174,0xE49B69C1,0xEFBE4786,0xFC19DC6,0x240CA1CC,0x2DE92C6F,0x4A7484AA,0x5CB0A9DC,0x76F988DA,0x983E5152,0xA831C66D,0xB00327C8,0xBF597FC7,0xC6E00BF3,0xD5A79147,0x6CA6351,0x14292967,0x27B70A85,0x2E1B2138,0x4D2C6DFC,0x53380D13,0x650A7354,0x766A0ABB,0x81C2C92E,0x92722C85,0xA2BFE8A1,0xA81A664B,0xC24B8B70,0xC76C51A3,0xD192E819,0xD6990624,0xF40E3585,0x106AA070,0x19A4C116,0x1E376C08,0x2748774C,0x34B0BCB5,0x391C0CB3,0x4ED8AA4A,0x5B9CCA4F,0x682E6FF3,0x748F82EE,0x78A5636F,0x84C87814,0x8CC70208,0x90BEFFFA,0xA4506CEB,0xBEF9A3F7,0xC67178F2]; - var HASH = [0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19]; - var W = newArray(64); - var a, b, c, d, e, f, g, h, i, j; - var T1, T2; - /* append padding */ - m[l >> 5] |= 0x80 << (24 - l % 32); - m[((l + 64 >> 9) << 4) + 15] = l; - for ( var i = 0; i>5] |= (str.charCodeAt(i / chrsz) & mask) << (24 - i%32); - return bin; - } - function binb2hex (binarray) { - var hexcase = 0; /* hex output format. 0 - lowercase; 1 - uppercase */ - var hex_tab = hexcase ? "0123456789ABCDEF" : "0123456789abcdef"; - var str = ""; - for (var i = 0; i < binarray.length * 4; i++) { - str += hex_tab.charAt((binarray[i>>2] >> ((3 - i%4)*8+4)) & 0xF) + - hex_tab.charAt((binarray[i>>2] >> ((3 - i%4)*8 )) & 0xF); - } - return str; - } - function hex_sha256(s){ - return binb2hex(core_sha256(str2binb(s),s.length * chrsz)); - } - module.exports.hex_sha256 = hex_sha256; -}()); - -}, -"Common.js": function(module, exports, require){ -/* - * Copyright 2014 XWiki SAS - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -var PARANOIA = module.exports.PARANOIA = false; - -/* throw errors over non-compliant messages which would otherwise be treated as invalid */ -var TESTING = module.exports.TESTING = true; - -var assert = module.exports.assert = function (expr) { - if (!expr) { throw new Error("Failed assertion"); } -}; - -var isUint = module.exports.isUint = function (integer) { - return (typeof(integer) === 'number') && - (Math.floor(integer) === integer) && - (integer >= 0); -}; - -var randomASCII = module.exports.randomASCII = function (length) { - var content = []; - for (var i = 0; i < length; i++) { - content[i] = String.fromCharCode( Math.floor(Math.random()*256) % 57 + 65 ); - } - return content.join(''); -}; - -var strcmp = module.exports.strcmp = function (a, b) { - if (PARANOIA && typeof(a) !== 'string') { throw new Error(); } - if (PARANOIA && typeof(b) !== 'string') { throw new Error(); } - return ( (a === b) ? 0 : ( (a > b) ? 1 : -1 ) ); -} - -}, -"Message.js": function(module, exports, require){ -/* - * Copyright 2014 XWiki SAS - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -var Common = require('./Common'); -var Operation = require('./Operation'); -var Patch = require('./Patch'); -var Sha = require('./SHA256'); - -var Message = module.exports; - -var REGISTER = Message.REGISTER = 0; -var REGISTER_ACK = Message.REGISTER_ACK = 1; -var PATCH = Message.PATCH = 2; -var DISCONNECT = Message.DISCONNECT = 3; -var PING = Message.PING = 4; -var PONG = Message.PONG = 5; - -var check = Message.check = function(msg) { - Common.assert(msg.type === 'Message'); - Common.assert(typeof(msg.userName) === 'string'); - Common.assert(typeof(msg.authToken) === 'string'); - Common.assert(typeof(msg.channelId) === 'string'); - - if (msg.messageType === PATCH) { - Patch.check(msg.content); - Common.assert(typeof(msg.lastMsgHash) === 'string'); - } else if (msg.messageType === PING || msg.messageType === PONG) { - Common.assert(typeof(msg.lastMsgHash) === 'undefined'); - Common.assert(typeof(msg.content) === 'number'); - } else if (msg.messageType === REGISTER - || msg.messageType === REGISTER_ACK - || msg.messageType === DISCONNECT) - { - Common.assert(typeof(msg.lastMsgHash) === 'undefined'); - Common.assert(typeof(msg.content) === 'undefined'); - } else { - throw new Error("invalid message type [" + msg.messageType + "]"); - } -}; - -var create = Message.create = function (userName, authToken, channelId, type, content, lastMsgHash) { - var msg = { - type: 'Message', - userName: userName, - authToken: authToken, - channelId: channelId, - messageType: type, - content: content, - lastMsgHash: lastMsgHash - }; - if (Common.PARANOIA) { check(msg); } - return msg; -}; - -var toString = Message.toString = function (msg) { - if (Common.PARANOIA) { check(msg); } - var prefix = msg.messageType + ':'; - var content = ''; - if (msg.messageType === REGISTER) { - content = JSON.stringify([REGISTER]); - } else if (msg.messageType === PING || msg.messageType === PONG) { - content = JSON.stringify([msg.messageType, msg.content]); - } else if (msg.messageType === PATCH) { - content = JSON.stringify([PATCH, Patch.toObj(msg.content), msg.lastMsgHash]); - } - return msg.authToken.length + ":" + msg.authToken + - msg.userName.length + ":" + msg.userName + - msg.channelId.length + ":" + msg.channelId + - content.length + ':' + content; -}; - -var fromString = Message.fromString = function (str) { - var msg = str; - - var unameLen = msg.substring(0,msg.indexOf(':')); - msg = msg.substring(unameLen.length+1); - var userName = msg.substring(0,Number(unameLen)); - msg = msg.substring(userName.length); - - var channelIdLen = msg.substring(0,msg.indexOf(':')); - msg = msg.substring(channelIdLen.length+1); - var channelId = msg.substring(0,Number(channelIdLen)); - msg = msg.substring(channelId.length); - - var contentStrLen = msg.substring(0,msg.indexOf(':')); - msg = msg.substring(contentStrLen.length+1); - var contentStr = msg.substring(0,Number(contentStrLen)); - - Common.assert(contentStr.length === Number(contentStrLen)); - - var content = JSON.parse(contentStr); - var message; - if (content[0] === PATCH) { - message = create(userName, '', channelId, PATCH, Patch.fromObj(content[1]), content[2]); - } else if (content[0] === PING || content[0] === PONG) { - message = create(userName, '', channelId, content[0], content[1]); - } else { - message = create(userName, '', channelId, content[0]); - } - - // This check validates every operation in the patch. - check(message); - - return message -}; - -var hashOf = Message.hashOf = function (msg) { - if (Common.PARANOIA) { check(msg); } - var authToken = msg.authToken; - msg.authToken = ''; - var hash = Sha.hex_sha256(toString(msg)); - msg.authToken = authToken; - return hash; -}; - -}, -"ChainPad.js": function(module, exports, require){ -/* - * Copyright 2014 XWiki SAS - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -var Common = require('./Common'); -var Operation = module.exports.Operation = require('./Operation'); -var Patch = require('./Patch'); -var Message = require('./Message'); -var Sha = require('./SHA256'); - -var ChainPad = {}; - -// hex_sha256('') -var EMPTY_STR_HASH = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; -var ZERO = '0000000000000000000000000000000000000000000000000000000000000000'; - -var enterChainPad = function (realtime, func) { - return function () { - if (realtime.failed) { return; } - func.apply(null, arguments); - }; -}; - -var debug = function (realtime, msg) { - console.log("[" + realtime.userName + "] " + msg); -}; - -var schedule = function (realtime, func, timeout) { - if (!timeout) { - timeout = Math.floor(Math.random() * 2 * realtime.avgSyncTime); - } - var to = setTimeout(enterChainPad(realtime, function () { - realtime.schedules.splice(realtime.schedules.indexOf(to), 1); - func(); - }), timeout); - realtime.schedules.push(to); - return to; -}; - -var unschedule = function (realtime, schedule) { - var index = realtime.schedules.indexOf(schedule); - if (index > -1) { - realtime.schedules.splice(index, 1); - } - clearTimeout(schedule); -}; - -var onMessage = function (realtime, message, callback) { - if (!realtime.messageHandlers.length) { - callback("no onMessage() handler registered"); - } - for (var i = 0; i < realtime.messageHandlers.length; i++) { - realtime.messageHandlers[i](message, function () { - callback.apply(null, arguments); - callback = function () { }; - }); - } -}; - -var sync = function (realtime) { - if (Common.PARANOIA) { check(realtime); } - if (realtime.syncSchedule) { - unschedule(realtime, realtime.syncSchedule); - realtime.syncSchedule = null; - } else { - // we're currently waiting on something from the server. - return; - } - - realtime.uncommitted = Patch.simplify( - realtime.uncommitted, realtime.authDoc, realtime.config.operationSimplify); - - if (realtime.uncommitted.operations.length === 0) { - //debug(realtime, "No data to sync to the server, sleeping"); - realtime.syncSchedule = schedule(realtime, function () { sync(realtime); }); - return; - } - - var msg; - if (realtime.best === realtime.initialMessage) { - msg = realtime.initialMessage; - } else { - msg = Message.create(realtime.userName, - realtime.authToken, - realtime.channelId, - Message.PATCH, - realtime.uncommitted, - realtime.best.hashOf); - } - - var strMsg = Message.toString(msg); - - onMessage(realtime, strMsg, function (err) { - if (err) { - debug(realtime, "Posting to server failed [" + err + "]"); - } - }); - - var hash = Message.hashOf(msg); - - var timeout = schedule(realtime, function () { - debug(realtime, "Failed to send message ["+hash+"] to server"); - sync(realtime); - }, 10000 + (Math.random() * 5000)); - realtime.pending = { - hash: hash, - callback: function () { - if (realtime.initialMessage && realtime.initialMessage.hashOf === hash) { - debug(realtime, "initial Ack received ["+hash+"]"); - realtime.initialMessage = null; - } - unschedule(realtime, timeout); - realtime.syncSchedule = schedule(realtime, function () { sync(realtime); }, 0); - } - }; - if (Common.PARANOIA) { check(realtime); } -}; - -var getMessages = function (realtime) { - realtime.registered = true; - /*var to = schedule(realtime, function () { - throw new Error("failed to connect to the server"); - }, 5000);*/ - var msg = Message.create(realtime.userName, - realtime.authToken, - realtime.channelId, - Message.REGISTER); - onMessage(realtime, Message.toString(msg), function (err) { - if (err) { throw err; } - }); -}; - -var sendPing = function (realtime) { - realtime.pingSchedule = undefined; - realtime.lastPingTime = (new Date()).getTime(); - var msg = Message.create(realtime.userName, - realtime.authToken, - realtime.channelId, - Message.PING, - realtime.lastPingTime); - onMessage(realtime, Message.toString(msg), function (err) { - if (err) { throw err; } - }); -}; - -var onPong = function (realtime, msg) { - if (Common.PARANOIA) { - Common.assert(realtime.lastPingTime === Number(msg.content)); - } - realtime.lastPingLag = (new Date()).getTime() - Number(msg.content); - realtime.lastPingTime = 0; - realtime.pingSchedule = - schedule(realtime, function () { sendPing(realtime); }, realtime.pingCycle); -}; - -var create = ChainPad.create = function (userName, authToken, channelId, initialState, config) { - - var realtime = { - type: 'ChainPad', - - authDoc: '', - - config: config || {}, - - userName: userName, - authToken: authToken, - channelId: channelId, - - /** A patch representing all uncommitted work. */ - uncommitted: null, - - uncommittedDocLength: initialState.length, - - patchHandlers: [], - opHandlers: [], - - messageHandlers: [], - - schedules: [], - - syncSchedule: null, - - registered: false, - - avgSyncTime: 100, - - // this is only used if PARANOIA is enabled. - userInterfaceContent: undefined, - - failed: false, - - // hash and callback for previously send patch, currently in flight. - pending: null, - - messages: {}, - messagesByParent: {}, - - rootMessage: null, - - /** - * Set to the message which sets the initialState if applicable. - * Reset to null after the initial message has been successfully broadcasted. - */ - initialMessage: null, - - userListChangeHandlers: [], - userList: [], - - /** The schedule() for sending pings. */ - pingSchedule: undefined, - - lastPingLag: 0, - lastPingTime: 0, - - /** Average number of milliseconds between pings. */ - pingCycle: 5000 - }; - - if (Common.PARANOIA) { - realtime.userInterfaceContent = initialState; - } - - var zeroPatch = Patch.create(EMPTY_STR_HASH); - zeroPatch.inverseOf = Patch.invert(zeroPatch, ''); - zeroPatch.inverseOf.inverseOf = zeroPatch; - var zeroMsg = Message.create('', '', channelId, Message.PATCH, zeroPatch, ZERO); - zeroMsg.hashOf = Message.hashOf(zeroMsg); - zeroMsg.parentCount = 0; - realtime.messages[zeroMsg.hashOf] = zeroMsg; - (realtime.messagesByParent[zeroMsg.lastMessageHash] || []).push(zeroMsg); - realtime.rootMessage = zeroMsg; - realtime.best = zeroMsg; - - if (initialState === '') { - realtime.uncommitted = Patch.create(zeroPatch.inverseOf.parentHash); - return realtime; - } - - var initialOp = Operation.create(0, 0, initialState); - var initialStatePatch = Patch.create(zeroPatch.inverseOf.parentHash); - Patch.addOperation(initialStatePatch, initialOp); - initialStatePatch.inverseOf = Patch.invert(initialStatePatch, ''); - initialStatePatch.inverseOf.inverseOf = initialStatePatch; - - // flag this patch so it can be handled specially. - // Specifically, we never treat an initialStatePatch as our own, - // we let it be reverted to prevent duplication of data. - initialStatePatch.isInitialStatePatch = true; - initialStatePatch.inverseOf.isInitialStatePatch = true; - - realtime.authDoc = initialState; - if (Common.PARANOIA) { - realtime.userInterfaceContent = initialState; - } - initialMessage = Message.create(realtime.userName, - realtime.authToken, - realtime.channelId, - Message.PATCH, - initialStatePatch, - zeroMsg.hashOf); - initialMessage.hashOf = Message.hashOf(initialMessage); - initialMessage.parentCount = 1; - - realtime.messages[initialMessage.hashOf] = initialMessage; - (realtime.messagesByParent[initialMessage.lastMessageHash] || []).push(initialMessage); - - realtime.best = initialMessage; - realtime.uncommitted = Patch.create(initialStatePatch.inverseOf.parentHash); - realtime.initialMessage = initialMessage; - - return realtime; -}; - -var getParent = function (realtime, message) { - return message.parent = message.parent || realtime.messages[message.lastMsgHash]; -}; - -var check = ChainPad.check = function(realtime) { - Common.assert(realtime.type === 'ChainPad'); - Common.assert(typeof(realtime.authDoc) === 'string'); - - Patch.check(realtime.uncommitted, realtime.authDoc.length); - - var uiDoc = Patch.apply(realtime.uncommitted, realtime.authDoc); - if (uiDoc.length !== realtime.uncommittedDocLength) { - Common.assert(0); - } - if (realtime.userInterfaceContent !== '') { - Common.assert(uiDoc === realtime.userInterfaceContent); - } - - var doc = realtime.authDoc; - var patchMsg = realtime.best; - Common.assert(patchMsg.content.inverseOf.parentHash === realtime.uncommitted.parentHash); - var patches = []; - do { - patches.push(patchMsg); - doc = Patch.apply(patchMsg.content.inverseOf, doc); - } while ((patchMsg = getParent(realtime, patchMsg))); - Common.assert(doc === ''); - while ((patchMsg = patches.pop())) { - doc = Patch.apply(patchMsg.content, doc); - } - Common.assert(doc === realtime.authDoc); -}; - -var doOperation = ChainPad.doOperation = function (realtime, op) { - if (Common.PARANOIA) { - check(realtime); - realtime.userInterfaceContent = Operation.apply(op, realtime.userInterfaceContent); - } - Operation.check(op, realtime.uncommittedDocLength); - Patch.addOperation(realtime.uncommitted, op); - realtime.uncommittedDocLength += Operation.lengthChange(op); -}; - -var isAncestorOf = function (realtime, ancestor, decendent) { - if (!decendent || !ancestor) { return false; } - if (ancestor === decendent) { return true; } - return isAncestorOf(realtime, ancestor, getParent(realtime, decendent)); -}; - -var parentCount = function (realtime, message) { - if (typeof(message.parentCount) !== 'number') { - message.parentCount = parentCount(realtime, getParent(realtime, message)) + 1; - } - return message.parentCount; -}; - -var applyPatch = function (realtime, author, patch) { - if (author === realtime.userName && !patch.isInitialStatePatch) { - var inverseOldUncommitted = Patch.invert(realtime.uncommitted, realtime.authDoc); - var userInterfaceContent = Patch.apply(realtime.uncommitted, realtime.authDoc); - if (Common.PARANOIA) { - Common.assert(userInterfaceContent === realtime.userInterfaceContent); - } - realtime.uncommitted = Patch.merge(inverseOldUncommitted, patch); - realtime.uncommitted = Patch.invert(realtime.uncommitted, userInterfaceContent); - - } else { - - // don't bother trying to JSON.parse things if authDoc is not defined - // you'll never get past that point if you throw errors - if (realtime.authDoc) { - try { - var applied = Patch.apply(realtime.uncommitted, realtime.authDoc); - if (applied) { - JSON.parse(applied); - } else { - - console.error("[patchApplyError] Patch.apply(realtime.uncommitted, realtime.authDoc) // falsey"); - console.log({ - uncommitted: realtime.uncommitted, - authDoc: realtime.authdoc - }); - //throw new Error("[patchApplyError] Patch.apply(realtime.uncommitted, realtime.authDoc) // falsey"); - } - - } catch (err) { - console.error('[patchApplyError] realtime.uncommited => authDoc'); - console.error(err); - console.log({ - applied: applied, - uncommitted: realtime.uncommitted, - authDoc: realtime.authdoc - }); - - //throw new Error(); - } - try { - JSON.parse(realtime.authDoc); - } catch (err) { - console.error('[patchApplyError] realtime.authDoc'); - console.error(err); - //throw new Error(); - } - } - - realtime.uncommitted = - Patch.transform( - realtime.uncommitted, patch, realtime.authDoc, realtime.config.transformFunction); - } - realtime.uncommitted.parentHash = patch.inverseOf.parentHash; - - var temp = Patch.apply(patch, realtime.authDoc); - // changed by ansuz - /* - try { - JSON.parse(temp); - } catch (err) { - console.log(err); - throw new Error(); - }*/ - - realtime.authDoc = temp; - if (Common.PARANOIA) { - realtime.userInterfaceContent = Patch.apply(realtime.uncommitted, realtime.authDoc); - } -}; - -var revertPatch = function (realtime, author, patch) { - applyPatch(realtime, author, patch.inverseOf); -}; - -var getBestChild = function (realtime, msg) { - var best = msg; - (realtime.messagesByParent[msg.hashOf] || []).forEach(function (child) { - Common.assert(child.lastMsgHash === msg.hashOf); - child = getBestChild(realtime, child); - if (parentCount(realtime, child) > parentCount(realtime, best)) { best = child; } - }); - return best; -}; - -var userListChange = function (realtime) { - for (var i = 0; i < realtime.userListChangeHandlers.length; i++) { - var list = []; - list.push.apply(list, realtime.userList); - realtime.userListChangeHandlers[i](list); - } -}; - -var handleMessage = ChainPad.handleMessage = function (realtime, msgStr) { - - if (Common.PARANOIA) { check(realtime); } - var msg = Message.fromString(msgStr); - Common.assert(msg.channelId === realtime.channelId); - - if (msg.messageType === Message.REGISTER_ACK) { - debug(realtime, "registered"); - realtime.registered = true; - sendPing(realtime); - return; - } - - if (msg.messageType === Message.REGISTER) { - realtime.userList.push(msg.userName); - userListChange(realtime); - return; - } - - if (msg.messageType === Message.PONG) { - onPong(realtime, msg); - return; - } - - if (msg.messageType === Message.DISCONNECT) { - if (msg.userName === '') { - realtime.userList = []; - userListChange(realtime); - return; - } - var idx = realtime.userList.indexOf(msg.userName); - if (Common.PARANOIA) { Common.assert(idx > -1); } - if (idx > -1) { - realtime.userList.splice(idx, 1); - userListChange(realtime); - } - return; - } - - // otherwise it's a disconnect. - if (msg.messageType !== Message.PATCH) { return; } - - msg.hashOf = Message.hashOf(msg); - - if (realtime.pending && realtime.pending.hash === msg.hashOf) { - realtime.pending.callback(); - realtime.pending = null; - } - - if (realtime.messages[msg.hashOf]) { - debug(realtime, "Patch [" + msg.hashOf + "] is already known"); - if (Common.PARANOIA) { check(realtime); } - return; - } - - realtime.messages[msg.hashOf] = msg; - (realtime.messagesByParent[msg.lastMsgHash] = - realtime.messagesByParent[msg.lastMsgHash] || []).push(msg); - - if (!isAncestorOf(realtime, realtime.rootMessage, msg)) { - // we'll probably find the missing parent later. - debug(realtime, "Patch [" + msg.hashOf + "] not connected to root"); - if (Common.PARANOIA) { check(realtime); } - return; - } - - // of this message fills in a hole in the chain which makes another patch better, swap to the - // best child of this patch since longest chain always wins. - msg = getBestChild(realtime, msg); - var patch = msg.content; - - // Find the ancestor of this patch which is in the main chain, reverting as necessary - var toRevert = []; - var commonAncestor = realtime.best; - if (!isAncestorOf(realtime, realtime.best, msg)) { - var pcBest = parentCount(realtime, realtime.best); - var pcMsg = parentCount(realtime, msg); - if (pcBest < pcMsg - || (pcBest === pcMsg - && Common.strcmp(realtime.best.hashOf, msg.hashOf) > 0)) - { - // switch chains - while (commonAncestor && !isAncestorOf(realtime, commonAncestor, msg)) { - toRevert.push(commonAncestor); - commonAncestor = getParent(realtime, commonAncestor); - } - Common.assert(commonAncestor); - } else { - debug(realtime, "Patch [" + msg.hashOf + "] chain is ["+pcMsg+"] best chain is ["+pcBest+"]"); - if (Common.PARANOIA) { check(realtime); } - return; - } - } - - // Find the parents of this patch which are not in the main chain. - var toApply = []; - var current = msg; - do { - toApply.unshift(current); - current = getParent(realtime, current); - Common.assert(current); - } while (current !== commonAncestor); - - - var authDocAtTimeOfPatch = realtime.authDoc; - - for (var i = 0; i < toRevert.length; i++) { - authDocAtTimeOfPatch = Patch.apply(toRevert[i].content.inverseOf, authDocAtTimeOfPatch); - } - - // toApply.length-1 because we do not want to apply the new patch. - for (var i = 0; i < toApply.length-1; i++) { - if (typeof(toApply[i].content.inverseOf) === 'undefined') { - toApply[i].content.inverseOf = Patch.invert(toApply[i].content, authDocAtTimeOfPatch); - toApply[i].content.inverseOf.inverseOf = toApply[i].content; - } - authDocAtTimeOfPatch = Patch.apply(toApply[i].content, authDocAtTimeOfPatch); - } - - if (Sha.hex_sha256(authDocAtTimeOfPatch) !== patch.parentHash) { - debug(realtime, "patch [" + msg.hashOf + "] parentHash is not valid"); - if (Common.PARANOIA) { check(realtime); } - if (Common.TESTING) { throw new Error(); } - delete realtime.messages[msg.hashOf]; - return; - } - - var simplePatch = - Patch.simplify(patch, authDocAtTimeOfPatch, realtime.config.operationSimplify); - if (!Patch.equals(simplePatch, patch)) { - debug(realtime, "patch [" + msg.hashOf + "] can be simplified"); - if (Common.PARANOIA) { check(realtime); } - if (Common.TESTING) { throw new Error(); } - delete realtime.messages[msg.hashOf]; - return; - } - - patch.inverseOf = Patch.invert(patch, authDocAtTimeOfPatch); - patch.inverseOf.inverseOf = patch; - - realtime.uncommitted = Patch.simplify( - realtime.uncommitted, realtime.authDoc, realtime.config.operationSimplify); - var oldUserInterfaceContent = Patch.apply(realtime.uncommitted, realtime.authDoc); - if (Common.PARANOIA) { - Common.assert(oldUserInterfaceContent === realtime.userInterfaceContent); - } - - // Derive the patch for the user's uncommitted work - var uncommittedPatch = Patch.invert(realtime.uncommitted, realtime.authDoc); - - for (var i = 0; i < toRevert.length; i++) { - debug(realtime, "reverting [" + toRevert[i].hashOf + "]"); - uncommittedPatch = Patch.merge(uncommittedPatch, toRevert[i].content.inverseOf); - revertPatch(realtime, toRevert[i].userName, toRevert[i].content); - } - - for (var i = 0; i < toApply.length; i++) { - debug(realtime, "applying [" + toApply[i].hashOf + "]"); - uncommittedPatch = Patch.merge(uncommittedPatch, toApply[i].content); - applyPatch(realtime, toApply[i].userName, toApply[i].content); - } - - uncommittedPatch = Patch.merge(uncommittedPatch, realtime.uncommitted); - uncommittedPatch = Patch.simplify( - uncommittedPatch, oldUserInterfaceContent, realtime.config.operationSimplify); - - realtime.uncommittedDocLength += Patch.lengthChange(uncommittedPatch); - realtime.best = msg; - - if (Common.PARANOIA) { - // apply the uncommittedPatch to the userInterface content. - var newUserInterfaceContent = Patch.apply(uncommittedPatch, oldUserInterfaceContent); - Common.assert(realtime.userInterfaceContent.length === realtime.uncommittedDocLength); - Common.assert(newUserInterfaceContent === realtime.userInterfaceContent); - } - - if (uncommittedPatch.operations.length) { - // push the uncommittedPatch out to the user interface. - for (var i = 0; i < realtime.patchHandlers.length; i++) { - realtime.patchHandlers[i](uncommittedPatch); - } - if (realtime.opHandlers.length) { - for (var i = uncommittedPatch.operations.length-1; i >= 0; i--) { - for (var j = 0; j < realtime.opHandlers.length; j++) { - realtime.opHandlers[j](uncommittedPatch.operations[i]); - } - } - } - } - if (Common.PARANOIA) { check(realtime); } -}; - -var wasEverState = function (content, realtime) { - Common.assert(typeof(content) === 'string'); - // without this we would never get true on the ^HEAD - if (realtime.authDoc === content) { - return true; - } - - var hash = Sha.hex_sha256(content); - - var patchMsg = realtime.best; - do { - if (patchMsg.content.parentHash === hash) { return true; } - } while ((patchMsg = getParent(realtime, patchMsg))); - return false; -}; - -var getDepthOfState = function (content, minDepth, realtime) { - Common.assert(typeof(content) === 'string'); - - // minimum depth is an optional argument which defaults to zero - var minDepth = minDepth || 0; - - if (minDepth === 0 && realtime.authDoc === content) { - return 0; - } - - var hash = Sha.hex_sha256(content); - - var patchMsg = realtime.best; - var depth = 0; - - do { - if (depth < minDepth) { - // you haven't exceeded the minimum depth - } else { - //console.log("Exceeded minimum depth"); - // you *have* exceeded the minimum depth - if (patchMsg.content.parentHash === hash) { - // you found it! - return depth + 1; - } - } - depth++; - } while ((patchMsg = getParent(realtime, patchMsg))); - return; -}; - -module.exports.create = function (userName, authToken, channelId, initialState, conf) { - Common.assert(typeof(userName) === 'string'); - Common.assert(typeof(authToken) === 'string'); - Common.assert(typeof(channelId) === 'string'); - Common.assert(typeof(initialState) === 'string'); - var realtime = ChainPad.create(userName, authToken, channelId, initialState, conf); - return { - onPatch: enterChainPad(realtime, function (handler) { - Common.assert(typeof(handler) === 'function'); - realtime.patchHandlers.push(handler); - }), - onRemove: enterChainPad(realtime, function (handler) { - Common.assert(typeof(handler) === 'function'); - realtime.opHandlers.unshift(function (op) { - if (op.toRemove > 0) { handler(op.offset, op.toRemove); } - }); - }), - onInsert: enterChainPad(realtime, function (handler) { - Common.assert(typeof(handler) === 'function'); - realtime.opHandlers.push(function (op) { - if (op.toInsert.length > 0) { handler(op.offset, op.toInsert); } - }); - }), - remove: enterChainPad(realtime, function (offset, numChars) { - doOperation(realtime, Operation.create(offset, numChars, '')); - }), - insert: enterChainPad(realtime, function (offset, str) { - doOperation(realtime, Operation.create(offset, 0, str)); - }), - onMessage: enterChainPad(realtime, function (handler) { - Common.assert(typeof(handler) === 'function'); - realtime.messageHandlers.push(handler); - }), - message: enterChainPad(realtime, function (message) { - handleMessage(realtime, message); - }), - start: enterChainPad(realtime, function () { - getMessages(realtime); - if (realtime.syncSchedule) { unschedule(realtime, realtime.syncSchedule); } - realtime.syncSchedule = schedule(realtime, function () { sync(realtime); }); - }), - abort: enterChainPad(realtime, function () { - realtime.schedules.forEach(function (s) { clearTimeout(s) }); - }), - sync: enterChainPad(realtime, function () { - sync(realtime); - }), - getAuthDoc: function () { return realtime.authDoc; }, - getUserDoc: function () { return Patch.apply(realtime.uncommitted, realtime.authDoc); }, - onUserListChange: enterChainPad(realtime, function (handler) { - Common.assert(typeof(handler) === 'function'); - realtime.userListChangeHandlers.push(handler); - }), - getLag: function () { - if (realtime.lastPingTime) { - return { waiting:1, lag: (new Date()).getTime() - realtime.lastPingTime }; - } - return { waiting:0, lag: realtime.lastPingLag }; - }, - wasEverState: function (content) { - return wasEverState(content, realtime); - }, - getDepthOfState: function (content, minDepth) { - return getDepthOfState(content, minDepth, realtime); - } - }; -}; - -}, -"Operation.js": function(module, exports, require){ -/* - * Copyright 2014 XWiki SAS - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -var Common = require('./Common'); - -var Operation = module.exports; - -var check = Operation.check = function (op, docLength_opt) { - Common.assert(op.type === 'Operation'); - Common.assert(Common.isUint(op.offset)); - Common.assert(Common.isUint(op.toRemove)); - Common.assert(typeof(op.toInsert) === 'string'); - Common.assert(op.toRemove > 0 || op.toInsert.length > 0); - Common.assert(typeof(docLength_opt) !== 'number' || op.offset + op.toRemove <= docLength_opt); -}; - -var create = Operation.create = function (offset, toRemove, toInsert) { - var out = { - type: 'Operation', - offset: offset || 0, - toRemove: toRemove || 0, - toInsert: toInsert || '', - }; - if (Common.PARANOIA) { check(out); } - return out; -}; - -var toObj = Operation.toObj = function (op) { - if (Common.PARANOIA) { check(op); } - return [op.offset,op.toRemove,op.toInsert]; -}; - -var fromObj = Operation.fromObj = function (obj) { - Common.assert(Array.isArray(obj) && obj.length === 3); - return create(obj[0], obj[1], obj[2]); -}; - -var clone = Operation.clone = function (op) { - return create(op.offset, op.toRemove, op.toInsert); -}; - -/** - * @param op the operation to apply. - * @param doc the content to apply the operation on - */ -var apply = Operation.apply = function (op, doc) -{ - if (Common.PARANOIA) { - check(op); - Common.assert(typeof(doc) === 'string'); - Common.assert(op.offset + op.toRemove <= doc.length); - } - return doc.substring(0,op.offset) + op.toInsert + doc.substring(op.offset + op.toRemove); -}; - -var invert = Operation.invert = function (op, doc) { - if (Common.PARANOIA) { - check(op); - Common.assert(typeof(doc) === 'string'); - Common.assert(op.offset + op.toRemove <= doc.length); - } - var rop = clone(op); - rop.toInsert = doc.substring(op.offset, op.offset + op.toRemove); - rop.toRemove = op.toInsert.length; - return rop; -}; - -var simplify = Operation.simplify = function (op, doc) { - if (Common.PARANOIA) { - check(op); - Common.assert(typeof(doc) === 'string'); - Common.assert(op.offset + op.toRemove <= doc.length); - } - var rop = invert(op, doc); - op = clone(op); - - var minLen = Math.min(op.toInsert.length, rop.toInsert.length); - var i; - for (i = 0; i < minLen && rop.toInsert[i] === op.toInsert[i]; i++) ; - op.offset += i; - op.toRemove -= i; - op.toInsert = op.toInsert.substring(i); - rop.toInsert = rop.toInsert.substring(i); - - if (rop.toInsert.length === op.toInsert.length) { - for (i = rop.toInsert.length-1; i >= 0 && rop.toInsert[i] === op.toInsert[i]; i--) ; - op.toInsert = op.toInsert.substring(0, i+1); - op.toRemove = i+1; - } - - if (op.toRemove === 0 && op.toInsert.length === 0) { return null; } - return op; -}; - -var equals = Operation.equals = function (opA, opB) { - return (opA.toRemove === opB.toRemove - && opA.toInsert === opB.toInsert - && opA.offset === opB.offset); -}; - -var lengthChange = Operation.lengthChange = function (op) -{ - if (Common.PARANOIA) { check(op); } - return op.toInsert.length - op.toRemove; -}; - -/* - * @return the merged operation OR null if the result of the merger is a noop. - */ -var merge = Operation.merge = function (oldOpOrig, newOpOrig) { - if (Common.PARANOIA) { - check(newOpOrig); - check(oldOpOrig); - } - - var newOp = clone(newOpOrig); - var oldOp = clone(oldOpOrig); - var offsetDiff = newOp.offset - oldOp.offset; - - if (newOp.toRemove > 0) { - var origOldInsert = oldOp.toInsert; - oldOp.toInsert = ( - oldOp.toInsert.substring(0,offsetDiff) - + oldOp.toInsert.substring(offsetDiff + newOp.toRemove) - ); - newOp.toRemove -= (origOldInsert.length - oldOp.toInsert.length); - if (newOp.toRemove < 0) { newOp.toRemove = 0; } - - oldOp.toRemove += newOp.toRemove; - newOp.toRemove = 0; - } - - if (offsetDiff < 0) { - oldOp.offset += offsetDiff; - oldOp.toInsert = newOp.toInsert + oldOp.toInsert; - - } else if (oldOp.toInsert.length === offsetDiff) { - oldOp.toInsert = oldOp.toInsert + newOp.toInsert; - - } else if (oldOp.toInsert.length > offsetDiff) { - oldOp.toInsert = ( - oldOp.toInsert.substring(0,offsetDiff) - + newOp.toInsert - + oldOp.toInsert.substring(offsetDiff) - ); - } else { - throw new Error("should never happen\n" + - JSON.stringify([oldOpOrig,newOpOrig], null, ' ')); - } - - if (oldOp.toInsert === '' && oldOp.toRemove === 0) { - return null; - } - if (Common.PARANOIA) { check(oldOp); } - - return oldOp; -}; - -/** - * If the new operation deletes what the old op inserted or inserts content in the middle of - * the old op's content or if they abbut one another, they should be merged. - */ -var shouldMerge = Operation.shouldMerge = function (oldOp, newOp) { - if (Common.PARANOIA) { - check(oldOp); - check(newOp); - } - if (newOp.offset < oldOp.offset) { - return (oldOp.offset <= (newOp.offset + newOp.toRemove)); - } else { - return (newOp.offset <= (oldOp.offset + oldOp.toInsert.length)); - } -}; - -/** - * Rebase newOp against oldOp. - * - * @param oldOp the eariler operation to have happened. - * @param newOp the later operation to have happened (in time). - * @return either the untouched newOp if it need not be rebased, - * the rebased clone of newOp if it needs rebasing, or - * null if newOp and oldOp must be merged. - */ -var rebase = Operation.rebase = function (oldOp, newOp) { - if (Common.PARANOIA) { - check(oldOp); - check(newOp); - } - if (newOp.offset < oldOp.offset) { return newOp; } - newOp = clone(newOp); - newOp.offset += oldOp.toRemove; - newOp.offset -= oldOp.toInsert.length; - return newOp; -}; - -/** - * this is a lossy and dirty algorithm, everything else is nice but transformation - * has to be lossy because both operations have the same base and they diverge. - * This could be made nicer and/or tailored to a specific data type. - * - * @param toTransform the operation which is converted *MUTATED*. - * @param transformBy an existing operation which also has the same base. - * @return toTransform *or* null if the result is a no-op. - */ - -var transform0 = Operation.transform0 = function (text, toTransformOrig, transformByOrig) { - // Cloning the original transformations makes this algorithm such that it - // **DOES NOT MUTATE ANYMORE** - var toTransform = Operation.clone(toTransformOrig); - var transformBy = Operation.clone(transformByOrig); - - if (toTransform.offset > transformBy.offset) { - if (toTransform.offset > transformBy.offset + transformBy.toRemove) { - // simple rebase - toTransform.offset -= transformBy.toRemove; - toTransform.offset += transformBy.toInsert.length; - return toTransform; - } - // goto the end, anything you deleted that they also deleted should be skipped. - var newOffset = transformBy.offset + transformBy.toInsert.length; - toTransform.toRemove = 0; //-= (newOffset - toTransform.offset); - if (toTransform.toRemove < 0) { toTransform.toRemove = 0; } - toTransform.offset = newOffset; - if (toTransform.toInsert.length === 0 && toTransform.toRemove === 0) { - return null; - } - return toTransform; - } - if (toTransform.offset + toTransform.toRemove < transformBy.offset) { - return toTransform; - } - toTransform.toRemove = transformBy.offset - toTransform.offset; - if (toTransform.toInsert.length === 0 && toTransform.toRemove === 0) { - return null; - } - return toTransform; -}; - -/** - * @param toTransform the operation which is converted - * @param transformBy an existing operation which also has the same base. - * @return a modified clone of toTransform *or* toTransform itself if no change was made. - */ -var transform = Operation.transform = function (text, toTransform, transformBy, transformFunction) { - if (Common.PARANOIA) { - check(toTransform); - check(transformBy); - } - transformFunction = transformFunction || transform0; - toTransform = clone(toTransform); - var result = transformFunction(text, toTransform, transformBy); - if (Common.PARANOIA && result) { check(result); } - return result; -}; - -/** Used for testing. */ -var random = Operation.random = function (docLength) { - Common.assert(Common.isUint(docLength)); - var offset = Math.floor(Math.random() * 100000000 % docLength) || 0; - var toRemove = Math.floor(Math.random() * 100000000 % (docLength - offset)) || 0; - var toInsert = ''; - do { - var toInsert = Common.randomASCII(Math.floor(Math.random() * 20)); - } while (toRemove === 0 && toInsert === ''); - return create(offset, toRemove, toInsert); -}; - -} -}; -ChainPad = r("ChainPad.js");}()); diff --git a/www/_socket/realtime-input.js b/www/_socket/realtime-input.js index 24e2e140f..a8d02e0b5 100644 --- a/www/_socket/realtime-input.js +++ b/www/_socket/realtime-input.js @@ -20,7 +20,7 @@ define([ '/common/crypto.js', '/_socket/toolbar.js', '/_socket/text-patcher.js', - '/_socket/chainpad.js', + '/common/chainpad.js', '/bower_components/jquery/dist/jquery.min.js', ], function (Messages,/*FIXME*/ ReconnectingWebSocket, Crypto, Toolbar, TextPatcher) { var $ = window.jQuery; From a1f9b1017597ba39ce0ccc32e2fd538ae7592950 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 10:02:11 +0200 Subject: [PATCH 47/69] remove offending line from chainpad entirely RTCHAINPAD-3 --- www/common/chainpad.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/www/common/chainpad.js b/www/common/chainpad.js index ba8a4d8ca..9d9dcc071 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -223,10 +223,6 @@ var transform = Patch.transform = function (origToTransform, transformBy, doc, t var toTransform = clone(origToTransform); var text = doc; for (var i = toTransform.operations.length-1; i >= 0; i--) { - /* this line caused diffs to be applied against the incorrect text - which resulted in bad merges that weren't noticeable until we - tried to patch json. */ - //text = Operation.apply(toTransform.operations[i], text); for (var j = transformBy.operations.length-1; j >= 0; j--) { toTransform.operations[i] = Operation.transform(text, toTransform.operations[i], From 2c34833d2c860c5f7fa9b5ba7c49335191cf3d63 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 12:27:33 +0200 Subject: [PATCH 48/69] break text-patcher's functionality into components text-patcher.js now exports diff, patch, log, and apply change in addition to the previous 'create' method. --- www/_socket/text-patcher.js | 100 ++++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 34 deletions(-) diff --git a/www/_socket/text-patcher.js b/www/_socket/text-patcher.js index 9f51c07b5..87cefa9dc 100644 --- a/www/_socket/text-patcher.js +++ b/www/_socket/text-patcher.js @@ -1,14 +1,12 @@ define(function () { -/* applyChange takes: - ctx: the context (aka the realtime) - oldval: the old value - newval: the new value +/* diff takes two strings, the old content, and the desired content + it returns the difference between these two strings in the form + of an 'Operation' (as defined in chainpad.js). - it performs a diff on the two values, and generates patches - which are then passed into `ctx.remove` and `ctx.insert` + diff is purely functional. */ -var applyChange = function(ctx, oldval, newval) { +var diff = function (oldval, newval) { // Strings are immutable and have reference equality. I think this test is O(1), so its worth doing. if (oldval === newval) { return; @@ -25,38 +23,66 @@ var applyChange = function(ctx, oldval, newval) { commonEnd++; } - var result; - - /* throw some assertions in here before dropping patches into the realtime - - */ + var toRemove; + var toInsert; + /* throw some assertions in here before dropping patches into the realtime */ if (oldval.length !== commonStart + commonEnd) { - if (ctx.localChange) { ctx.localChange(true); } - result = oldval.length - commonStart - commonEnd; - ctx.remove(commonStart, result); - console.log('removal at position: %s, length: %s', commonStart, result); - console.log("remove: [" + oldval.slice(commonStart, commonStart + result ) + ']'); + toRemove = oldval.length - commonStart - commonEnd; } if (newval.length !== commonStart + commonEnd) { - if (ctx.localChange) { ctx.localChange(true); } - result = newval.slice(commonStart, newval.length - commonEnd); - ctx.insert(commonStart, result); - console.log("insert: [" + result + "]"); + toInsert = newval.slice(commonStart, newval.length - commonEnd); } - var userDoc; - try { - var userDoc = ctx.getUserDoc(); - JSON.parse(userDoc); - } catch (err) { - console.error('[textPatcherParseErr]'); - console.error(err); - window.REALTIME_MODULE.textPatcher_parseError = { - error: err, - userDoc: userDoc - }; - } + return { + type: 'Operation', + offset: commonStart, + toInsert: toInsert, + toRemove: toRemove + }; +} + +/* patch accepts a realtime facade and an operation (which might be falsey) + it applies the operation to the realtime as components (remove/insert) + + patch has no return value, and operates solely through side effects on + the realtime facade. +*/ +var patch = function (ctx, op) { + if (!op) { return; } + if (op.toRemove) { ctx.remove(op.offset, op.toRemove); } + if (op.toInsert) { ctx.insert(op.offset, op.toInsert); } +}; + +/* log accepts a string and an operation, and prints an object to the console + the object will display the content which is to be removed, and the content + which will be inserted in its place. + + log is useful for debugging, but can otherwise be disabled. +*/ +var log = function (text, op) { + if (!op) { return; } + console.log({ + insert: op.toInsert, + remove: text.slice(op.offset, op.offset + op.toRemove) + }); +}; + +/* applyChange takes: + ctx: the context (aka the realtime) + oldval: the old value + newval: the new value + + it performs a diff on the two values, and generates patches + which are then passed into `ctx.remove` and `ctx.insert`. + + Due to its reliance on patch, applyChange has side effects on the supplied + realtime facade. +*/ +var applyChange = function(ctx, oldval, newval) { + var op = diff(oldval, newval); + // log(oldval, op) + patch(ctx, op); }; var create = function(config) { @@ -84,5 +110,11 @@ var create = function(config) { }; }; -return { create: create }; +return { + create: create, // create a TextPatcher object + diff: diff, // diff two strings + patch: patch, // apply an operation to a chainpad's realtime facade + log: log, // print the components of an operation + applyChange: applyChange // a convenient wrapper around diff/log/patch +}; }); From 9805958ad76637f8240acae8c48bdbf2c3fd2574 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 12:46:49 +0200 Subject: [PATCH 49/69] stabilize text-patcher.js into /common/TextPatcher.js --- www/_socket/realtime-input.js | 2 +- www/{_socket/text-patcher.js => common/TextPatcher.js} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename www/{_socket/text-patcher.js => common/TextPatcher.js} (100%) diff --git a/www/_socket/realtime-input.js b/www/_socket/realtime-input.js index a8d02e0b5..d9f2970e8 100644 --- a/www/_socket/realtime-input.js +++ b/www/_socket/realtime-input.js @@ -19,7 +19,7 @@ define([ '/bower_components/reconnectingWebsocket/reconnecting-websocket.js', '/common/crypto.js', '/_socket/toolbar.js', - '/_socket/text-patcher.js', + '/common/TextPatcher.js', '/common/chainpad.js', '/bower_components/jquery/dist/jquery.min.js', ], function (Messages,/*FIXME*/ ReconnectingWebSocket, Crypto, Toolbar, TextPatcher) { diff --git a/www/_socket/text-patcher.js b/www/common/TextPatcher.js similarity index 100% rename from www/_socket/text-patcher.js rename to www/common/TextPatcher.js From 39071021ebfddd4d618b44891550c0847bcbff8e Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 12:53:23 +0200 Subject: [PATCH 50/69] stabilize typingTest.js as /common/TypingTests.js --- www/_socket/main.js | 2 +- www/{_socket/typingTest.js => common/TypingTests.js} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename www/{_socket/typingTest.js => common/TypingTests.js} (100%) diff --git a/www/_socket/main.js b/www/_socket/main.js index f334929e9..27c2ef058 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -8,7 +8,7 @@ define([ '/_socket/toolbar.js', '/common/cursor.js', '/common/json-ot.js', - '/_socket/typingTest.js', + '/common/TypingTests.js', '/bower_components/diff-dom/diffDOM.js', '/bower_components/jquery/dist/jquery.min.js', '/customize/pad.js' diff --git a/www/_socket/typingTest.js b/www/common/TypingTests.js similarity index 100% rename from www/_socket/typingTest.js rename to www/common/TypingTests.js From 1a22592afa6e819fac80dfececc845668d58fd75 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 13:06:52 +0200 Subject: [PATCH 51/69] remove unused modules from realtime-input.js --- www/_socket/realtime-input.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/www/_socket/realtime-input.js b/www/_socket/realtime-input.js index d9f2970e8..edb8e3791 100644 --- a/www/_socket/realtime-input.js +++ b/www/_socket/realtime-input.js @@ -18,11 +18,10 @@ define([ '/common/messages.js', '/bower_components/reconnectingWebsocket/reconnecting-websocket.js', '/common/crypto.js', - '/_socket/toolbar.js', '/common/TextPatcher.js', '/common/chainpad.js', '/bower_components/jquery/dist/jquery.min.js', -], function (Messages,/*FIXME*/ ReconnectingWebSocket, Crypto, Toolbar, TextPatcher) { +], function (Messages, ReconnectingWebSocket, Crypto, TextPatcher) { var $ = window.jQuery; var ChainPad = window.ChainPad; var PARANOIA = true; From 6b9d982d40ceffd06cec4acfc0a0216b8a9bd8fb Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 13:10:57 +0200 Subject: [PATCH 52/69] stabilize _socket/realtime-input.js ...as common/RealtimeTextSocket.js --- www/_socket/main.js | 2 +- www/{_socket/realtime-input.js => common/RealtimeTextSocket.js} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename www/{_socket/realtime-input.js => common/RealtimeTextSocket.js} (100%) diff --git a/www/_socket/main.js b/www/_socket/main.js index 27c2ef058..bccdc99c6 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -2,7 +2,7 @@ define([ '/api/config?cb=' + Math.random().toString(16).substring(2), '/common/messages.js', '/common/crypto.js', - '/_socket/realtime-input.js', + '/common/RealtimeTextSocket.js', '/common/hyperjson.js', '/common/hyperscript.js', '/_socket/toolbar.js', diff --git a/www/_socket/realtime-input.js b/www/common/RealtimeTextSocket.js similarity index 100% rename from www/_socket/realtime-input.js rename to www/common/RealtimeTextSocket.js From 6bb37aed449f2b8674d27d2324201f54e0ca6b1b Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 14:05:56 +0200 Subject: [PATCH 53/69] main.js : support tab insertion in /hack/ pad --- www/hack/main.js | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/www/hack/main.js b/www/hack/main.js index ab38f8402..8877260eb 100644 --- a/www/hack/main.js +++ b/www/hack/main.js @@ -3,9 +3,10 @@ define([ '/common/realtime-input.js', '/common/messages.js', '/common/crypto.js', + '/common/cursor.js', '/bower_components/jquery/dist/jquery.min.js', '/customize/pad.js' -], function (Config, Realtime, Messages, Crypto) { +], function (Config, Realtime, Messages, Crypto, Cursor) { var $ = window.jQuery; $(window).on('hashchange', function() { window.location.reload(); @@ -59,6 +60,48 @@ define([ var rt = Realtime.start(config); + var cursor = Cursor($textarea[0]); + + var splice = function (str, index, chars) { + var count = chars.length; + return str.slice(0, index) + chars + str.slice((index -1) + count); + }; + + var setSelectionRange = function (input, start, end) { + if (input.setSelectionRange) { + input.focus(); + input.setSelectionRange(start, end); + } else if (input.createTextRange) { + var range = input.createTextRange(); + range.collapse(true); + range.moveEnd('character', end); + range.moveStart('character', start); + range.select(); + } + }; + + var setCursor = function (el, pos) { + setSelectionRange(el, pos, pos); + }; + + $textarea.on('keypress', function (e) { + switch (e.key) { + case 'Tab': + // insert a tab wherever the cursor is... + var position = $textarea.prop("selectionStart"); + if (typeof position !== 'undefined') { + $textarea.val(function (i, val) { + return splice(val, position, "\t"); + }); + setCursor($textarea[0], position +1); + } + // prevent default behaviour for tab + e.preventDefault(); + default: + break; + } + }); + $run.click(function (e) { e.preventDefault(); var content = $textarea.val(); From a07774e81ac8a6e39cf9686fbd21cf64cff13934 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 14:12:44 +0200 Subject: [PATCH 54/69] Implement tests for serialization ensure that complex DOM elements can serialize and deserialize without modifications RTWYSIWYG-54 > implement tests for components of the WYSIWYG editor --- www/assert/hyperjson.js | 124 ++++++++++++ www/assert/hyperscript.js | 400 ++++++++++++++++++++++++++++++++++++++ www/assert/index.html | 25 +++ www/assert/main.js | 50 +++++ 4 files changed, 599 insertions(+) create mode 100644 www/assert/hyperjson.js create mode 100644 www/assert/hyperscript.js create mode 100644 www/assert/index.html create mode 100644 www/assert/main.js diff --git a/www/assert/hyperjson.js b/www/assert/hyperjson.js new file mode 100644 index 000000000..90edb0616 --- /dev/null +++ b/www/assert/hyperjson.js @@ -0,0 +1,124 @@ +define([], function () { + // this makes recursing a lot simpler + var isArray = function (A) { + return Object.prototype.toString.call(A)==='[object Array]'; + }; + + var parseStyle = function(el){ + var style = el.style; + var output = {}; + for (var i = 0; i < style.length; ++i) { + var item = style.item(i); + output[item] = style[item]; + } + return output; + }; + + var callOnHyperJSON = function (hj, cb) { + var children; + + if (hj && hj[2]) { + children = hj[2].map(function (child) { + if (isArray(child)) { + // if the child is an array, recurse + return callOnHyperJSON(child, cb); + } else if (typeof (child) === 'string') { + // string nodes have leading and trailing quotes + return child.replace(/(^"|"$)/g,""); + } else { + // the above branches should cover all methods + // if we hit this, there is a problem + throw new Error(); + } + }); + } else { + children = []; + } + // this should return the top level element of your new DOM + return cb(hj[0], hj[1], children); + }; + + var classify = function (token) { + return '.' + token.trim(); + }; + + var isValidClass = function (x) { + if (x && /\S/.test(x)) { + return true; + } + }; + + var isTruthy = function (x) { + return x; + }; + + var DOM2HyperJSON = function(el, predicate, filter){ + if(!el.tagName && el.nodeType === Node.TEXT_NODE){ + return el.textContent; + } + if(!el.attributes){ + return; + } + if (predicate) { + if (!predicate(el)) { + // shortcircuit + return; + } + } + + var attributes = {}; + + var i = 0; + for(;i < el.attributes.length; i++){ + var attr = el.attributes[i]; + if(attr.name && attr.value){ + if(attr.name === "style"){ + attributes.style = parseStyle(el); + } + else{ + attributes[attr.name] = attr.value; + } + } + } + + // this should never be longer than three elements + var result = []; + + // get the element type, id, and classes of the element + // and push them to the result array + var sel = el.tagName; + + if(attributes.id){ + // we don't have to do much to validate IDs because the browser + // will only permit one id to exist + // unless we come across a strange browser in the wild + sel = sel +'#'+ attributes.id; + delete attributes.id; + } + result.push(sel); + + // second element of the array is the element attributes + result.push(attributes); + + // third element of the array is an array of child nodes + var children = []; + + // js hint complains if we use 'var' here + i = 0; + for(; i < el.childNodes.length; i++){ + children.push(DOM2HyperJSON(el.childNodes[i], predicate, filter)); + } + result.push(children.filter(isTruthy)); + + if (filter) { + return filter(result); + } else { + return result; + } + }; + + return { + fromDOM: DOM2HyperJSON, + callOn: callOnHyperJSON + }; +}); diff --git a/www/assert/hyperscript.js b/www/assert/hyperscript.js new file mode 100644 index 000000000..52282dcfc --- /dev/null +++ b/www/assert/hyperscript.js @@ -0,0 +1,400 @@ +define([], function () { + var Hyperscript; + +(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o + * Available under the MIT License + * ECMAScript compliant, uniform cross-browser split method + */ + +/** + * Splits a string into an array of strings using a regex or string separator. Matches of the + * separator are not included in the result array. However, if `separator` is a regex that contains + * capturing groups, backreferences are spliced into the result each time `separator` is matched. + * Fixes browser bugs compared to the native `String.prototype.split` and can be used reliably + * cross-browser. + * @param {String} str String to split. + * @param {RegExp|String} separator Regex or string to use for separating the string. + * @param {Number} [limit] Maximum number of items to include in the result array. + * @returns {Array} Array of substrings. + * @example + * + * // Basic use + * split('a b c d', ' '); + * // -> ['a', 'b', 'c', 'd'] + * + * // With limit + * split('a b c d', ' ', 2); + * // -> ['a', 'b'] + * + * // Backreferences in result array + * split('..word1 word2..', /([a-z]+)(\d+)/i); + * // -> ['..', 'word', '1', ' ', 'word', '2', '..'] + */ +module.exports = (function split(undef) { + + var nativeSplit = String.prototype.split, + compliantExecNpcg = /()??/.exec("")[1] === undef, + // NPCG: nonparticipating capturing group + self; + + self = function(str, separator, limit) { + // If `separator` is not a regex, use `nativeSplit` + if (Object.prototype.toString.call(separator) !== "[object RegExp]") { + return nativeSplit.call(str, separator, limit); + } + var output = [], + flags = (separator.ignoreCase ? "i" : "") + (separator.multiline ? "m" : "") + (separator.extended ? "x" : "") + // Proposed for ES6 + (separator.sticky ? "y" : ""), + // Firefox 3+ + lastLastIndex = 0, + // Make `global` and avoid `lastIndex` issues by working with a copy + separator = new RegExp(separator.source, flags + "g"), + separator2, match, lastIndex, lastLength; + str += ""; // Type-convert + if (!compliantExecNpcg) { + // Doesn't need flags gy, but they don't hurt + separator2 = new RegExp("^" + separator.source + "$(?!\\s)", flags); + } + /* Values for `limit`, per the spec: + * If undefined: 4294967295 // Math.pow(2, 32) - 1 + * If 0, Infinity, or NaN: 0 + * If positive number: limit = Math.floor(limit); if (limit > 4294967295) limit -= 4294967296; + * If negative number: 4294967296 - Math.floor(Math.abs(limit)) + * If other: Type-convert, then use the above rules + */ + limit = limit === undef ? -1 >>> 0 : // Math.pow(2, 32) - 1 + limit >>> 0; // ToUint32(limit) + while (match = separator.exec(str)) { + // `separator.lastIndex` is not reliable cross-browser + lastIndex = match.index + match[0].length; + if (lastIndex > lastLastIndex) { + output.push(str.slice(lastLastIndex, match.index)); + // Fix browsers whose `exec` methods don't consistently return `undefined` for + // nonparticipating capturing groups + if (!compliantExecNpcg && match.length > 1) { + match[0].replace(separator2, function() { + for (var i = 1; i < arguments.length - 2; i++) { + if (arguments[i] === undef) { + match[i] = undef; + } + } + }); + } + if (match.length > 1 && match.index < str.length) { + Array.prototype.push.apply(output, match.slice(1)); + } + lastLength = match[0].length; + lastLastIndex = lastIndex; + if (output.length >= limit) { + break; + } + } + if (separator.lastIndex === match.index) { + separator.lastIndex++; // Avoid an infinite loop + } + } + if (lastLastIndex === str.length) { + if (lastLength || !separator.test("")) { + output.push(""); + } + } else { + output.push(str.slice(lastLastIndex)); + } + return output.length > limit ? output.slice(0, limit) : output; + }; + + return self; +})(); + +},{}],3:[function(require,module,exports){ +// contains, add, remove, toggle +var indexof = require('indexof') + +module.exports = ClassList + +function ClassList(elem) { + var cl = elem.classList + + if (cl) { + return cl + } + + var classList = { + add: add + , remove: remove + , contains: contains + , toggle: toggle + , toString: $toString + , length: 0 + , item: item + } + + return classList + + function add(token) { + var list = getTokens() + if (indexof(list, token) > -1) { + return + } + list.push(token) + setTokens(list) + } + + function remove(token) { + var list = getTokens() + , index = indexof(list, token) + + if (index === -1) { + return + } + + list.splice(index, 1) + setTokens(list) + } + + function contains(token) { + return indexof(getTokens(), token) > -1 + } + + function toggle(token) { + if (contains(token)) { + remove(token) + return false + } else { + add(token) + return true + } + } + + function $toString() { + return elem.className + } + + function item(index) { + var tokens = getTokens() + return tokens[index] || null + } + + function getTokens() { + var className = elem.className + + return filter(className.split(" "), isTruthy) + } + + function setTokens(list) { + var length = list.length + + elem.className = list.join(" ") + classList.length = length + + for (var i = 0; i < list.length; i++) { + classList[i] = list[i] + } + + delete list[length] + } +} + +function filter (arr, fn) { + var ret = [] + for (var i = 0; i < arr.length; i++) { + if (fn(arr[i])) ret.push(arr[i]) + } + return ret +} + +function isTruthy(value) { + return !!value +} + +},{"indexof":4}],4:[function(require,module,exports){ + +var indexOf = [].indexOf; + +module.exports = function(arr, obj){ + if (indexOf) return arr.indexOf(obj); + for (var i = 0; i < arr.length; ++i) { + if (arr[i] === obj) return i; + } + return -1; +}; +},{}],5:[function(require,module,exports){ +var h = require("./index.js"); + +module.exports = h; + +/* +$(function () { + + var newDoc = h('p', + + h('ul', 'bang bang bang'.split(/\s/).map(function (word) { + return h('li', word); + })) + ); + $('body').html(newDoc.outerHTML); +}); + +*/ + +},{"./index.js":1}],6:[function(require,module,exports){ + +},{}]},{},[5]); + + return Hyperscript; +}); diff --git a/www/assert/index.html b/www/assert/index.html new file mode 100644 index 000000000..5feebf5ae --- /dev/null +++ b/www/assert/index.html @@ -0,0 +1,25 @@ + + + + + + + + + +

Serialization tests

+ +

Test 1

+

class strings

+ +

pewpewpew

+ +
+ +

Test 2

+

XWiki Macros

+ + +

Here is a macro

+ +
diff --git a/www/assert/main.js b/www/assert/main.js new file mode 100644 index 000000000..f8c4f7ce1 --- /dev/null +++ b/www/assert/main.js @@ -0,0 +1,50 @@ +define([ + '/bower_components/jquery/dist/jquery.min.js', + '/assert/hyperjson.js', // serializing classes as an attribute + '/assert/hyperscript.js', // using setAttribute + '/common/TextPatcher.js' +], function (jQuery, Hyperjson, Hyperscript, TextPatcher) { + var $ = window.jQuery; + window.Hyperjson = Hyperjson; + window.Hyperscript = Hyperscript; + window.TextPatcher = TextPatcher; + + var assertions = 0; + + var assert = function (test, msg) { + if (test()) { + assertions++; + } else { + throw new Error(msg || ''); + } + }; + + var $body = $('body'); + + var roundTrip = function (target) { + assert(function () { + var hjson = Hyperjson.fromDOM(target); + var cloned = Hyperjson.callOn(hjson, Hyperscript); + + var success = cloned.outerHTML === target.outerHTML; + + if (!success) { + window.DEBUG = { + error: "Expected equality between A and B", + A: target.outerHTML, + B: cloned.outerHTML, + target: target, + diff: TextPatcher.diff(target.outerHTML, cloned.outerHTML) + }; + console.log(JSON.stringify(window.DEBUG, null, 2)); + } + + return success; + }, "Round trip serialization introduced artifacts."); + }; + + roundTrip($('#target')[0]); + roundTrip($('#widget')[0]); + + console.log("%s test%s passed", assertions, assertions === 1? '':'s'); +}); From 400f6efde15978f2755c8bc84bd0dc87a811e4d2 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 15:00:46 +0200 Subject: [PATCH 55/69] json-ot.js : jshint compliance --- www/common/json-ot.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/www/common/json-ot.js b/www/common/json-ot.js index 98e1af4ec..d8d75b7b6 100644 --- a/www/common/json-ot.js +++ b/www/common/json-ot.js @@ -5,15 +5,16 @@ define([ var JsonOT = {}; var validate = JsonOT.validate = function (text, toTransform, transformBy) { + var resultOp, text2, text3; try { // text = O (mutual common ancestor) // toTransform = A (the first incoming operation) // transformBy = B (the second incoming operation) // threeway merge (0, A, B) - var resultOp = ChainPad.Operation.transform0(text, toTransform, transformBy); - var text2 = ChainPad.Operation.apply(transformBy, text); - var text3 = ChainPad.Operation.apply(resultOp, text2); + resultOp = ChainPad.Operation.transform0(text, toTransform, transformBy); + text2 = ChainPad.Operation.apply(transformBy, text); + text3 = ChainPad.Operation.apply(resultOp, text2); try { JSON.parse(text3); return resultOp; @@ -35,8 +36,7 @@ define([ } } catch (x) { console.error(x); - console.error(e); - var info = window.REALTIME_MODULE.ot_applyError = { + window.REALTIME_MODULE.ot_applyError = { type: 'resultParseError', resultOp: resultOp, @@ -46,7 +46,7 @@ define([ text1: text, text2: text2, text3: text3, - error: e + error: x }; console.log('Debugging info available at `window.REALTIME_MODULE.ot_applyError`'); } From 851ccfbdb61929b3c888944585847eeb70f0ead4 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 15:35:07 +0200 Subject: [PATCH 56/69] detect falsey operations in OT and ignore them --- www/common/json-ot.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/www/common/json-ot.js b/www/common/json-ot.js index d8d75b7b6..a785d524c 100644 --- a/www/common/json-ot.js +++ b/www/common/json-ot.js @@ -4,6 +4,9 @@ define([ var ChainPad = window.ChainPad; var JsonOT = {}; +/* FIXME + resultOp after transform0() might be null, in which case you should return null + because it is simply a transformation which yields a "do nothing" operation */ var validate = JsonOT.validate = function (text, toTransform, transformBy) { var resultOp, text2, text3; try { @@ -13,6 +16,11 @@ define([ // threeway merge (0, A, B) resultOp = ChainPad.Operation.transform0(text, toTransform, transformBy); + + /* if after operational transform we find that no op is necessary + return null to ignore this patch */ + if (!resultOp) { return null; } + text2 = ChainPad.Operation.apply(transformBy, text); text3 = ChainPad.Operation.apply(resultOp, text2); try { From 2fdb4c12557466c192fef72fea7e3b4af2cfad1c Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 15:07:31 +0200 Subject: [PATCH 57/69] ignore external files and anything relying on es6 merge differences from netflux Conflicts: .jshintignore --- .jshintignore | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.jshintignore b/.jshintignore index 29b53f0f4..3780e2719 100644 --- a/.jshintignore +++ b/.jshintignore @@ -1,6 +1,6 @@ node_modules/ www/bower_components/ -www/code/codemirror-5.7/ +www/code/codemirror* www/code/mode/ www/code/codemirror.js www/pad/rangy.js @@ -12,3 +12,14 @@ storage/kad.js www/common/otaml.js www/common/diffDOM.js www/common/netflux.js + +www/padrtc +www/common/netflux-client.js +www/common/es6-promise.min.js +www/_pad + +NetFluxWebsocketSrv.js +NetFluxWebsocketServer.js +WebRTCSrv.js + +www/assert/hyperscript.js From 368d72b337bd7d4981a249d55be4dc0271401d80 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 14:51:06 +0200 Subject: [PATCH 58/69] add missing semicolons to TextPatcher.js --- www/common/TextPatcher.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/www/common/TextPatcher.js b/www/common/TextPatcher.js index 87cefa9dc..66858826d 100644 --- a/www/common/TextPatcher.js +++ b/www/common/TextPatcher.js @@ -40,7 +40,7 @@ var diff = function (oldval, newval) { toInsert: toInsert, toRemove: toRemove }; -} +}; /* patch accepts a realtime facade and an operation (which might be falsey) it applies the operation to the realtime as components (remove/insert) @@ -94,7 +94,7 @@ var create = function(config) { // *** remote -> local changes ctx.onPatch(function(pos, length) { - content = ctx.getUserDoc() + content = ctx.getUserDoc(); }); // propogate() From 2ffa69dc151915d7183f3023c4e9d915b9deb3bf Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 15:55:03 +0200 Subject: [PATCH 59/69] remove checks for a non-existent race condition --- www/_socket/main.js | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index bccdc99c6..67bfaff89 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -28,7 +28,6 @@ define([ toolbar; var module = window.REALTIME_MODULE = { - localChangeInProgress: 0, Hyperjson: Hyperjson, Hyperscript: Hyperscript }; @@ -187,23 +186,9 @@ define([ var DD = new DiffDom(diffOptions); - var localWorkInProgress = function (stage) { - if (module.localChangeInProgress) { - console.error("Applied a change while a local patch was in progress"); - console.error("local work was interrupted at stage: " + stage); - return true; - } - return false; - }; - // apply patches, and try not to lose the cursor in the process! var applyHjson = function (shjson) { - - localWorkInProgress(1); // check if this would interrupt local work - var userDocStateDom = hjsonToDom(JSON.parse(shjson)); - localWorkInProgress(2); // check again - /* in the DOM contentEditable is "false" while "contenteditable" is undefined. @@ -219,11 +204,8 @@ define([ this comes from hyperscript line 101. FIXME maybe */ userDocStateDom.setAttribute("contenteditable", "true"); // lol wtf - localWorkInProgress(3); // check again var patch = (DD).diff(inner, userDocStateDom); - localWorkInProgress(4); // check again (DD).apply(inner, patch); - localWorkInProgress(5); // check again }; var initializing = true; @@ -231,8 +213,6 @@ define([ var onRemote = realtimeOptions.onRemote = function (info) { if (initializing) { return; } - localWorkInProgress(0); - var shjson = info.realtime.getUserDoc(); // remember where the cursor is @@ -290,20 +270,11 @@ define([ the code less extensible. */ var propogate = rti.onLocal = function () { - /* if the problem were a matter of external patches being - applied while a local patch were in progress, then we would - expect to be able to check and find - 'module.localChangeInProgress' with a non-zero value while - we were applying a remote change. - */ - module.localChangeInProgress += 1; var shjson = JSON.stringify(Hyperjson.fromDOM(inner, isNotMagicLine, brFilter)); if (!rti.patchText(shjson)) { - module.localChangeInProgress -= 1; return; } rti.onEvent(shjson); - module.localChangeInProgress -= 1; }; /* hitting enter makes a new line, but places the cursor inside From 9f0cc4ed64f92c8ed8f951382efc901153e3f213 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 12 Apr 2016 18:51:03 +0200 Subject: [PATCH 60/69] update hack with a slightly better UI --- www/hack/index.html | 37 +++++++++++++++++++++++++++++-------- www/hack/main.js | 31 +++++++++++++++++++++++++------ 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/www/hack/index.html b/www/hack/index.html index 4fa7e6065..2e8df970f 100644 --- a/www/hack/index.html +++ b/www/hack/index.html @@ -12,8 +12,14 @@ box-sizing: border-box; } textarea{ + position: absolute; + top: 5vh; + left: 0px; + border: 0px; + + padding-top: 15px; width: 100%; - height: 100vh; + height: 95vh; max-width: 100%; max-height: 100vh; @@ -32,26 +38,41 @@ color: #637476; } - #run { + #panel { position: fixed; top: 0px; right: 0px; - - z-index: 100; - width: 5vw; + width: 100%; height: 5vh; + z-index: 95; + background-color: #777; + /* min-height: 75px; */ + } + #run { + display: block; + float: right; + height: 100%; + width: 10vw; + z-index: 100; + line-height: 5vw; + font-size: 1.5em; background-color: #222; color: #CCC; - - display: block; text-align: center; + border-radius: 5%; + border: 0px; } - RUN +
+ + + + RUN +
diff --git a/www/hack/main.js b/www/hack/main.js index 8877260eb..f5618c8b3 100644 --- a/www/hack/main.js +++ b/www/hack/main.js @@ -84,16 +84,35 @@ define([ setSelectionRange(el, pos, pos); }; + var state = {}; + + // TODO + $textarea.on('keydown', function (e) { + // track when control keys are pushed down + //switch (e.key) { } + }); + + // TODO + $textarea.on('keyup', function (e) { + // track when control keys are released + }); + $textarea.on('keypress', function (e) { switch (e.key) { case 'Tab': // insert a tab wherever the cursor is... - var position = $textarea.prop("selectionStart"); - if (typeof position !== 'undefined') { - $textarea.val(function (i, val) { - return splice(val, position, "\t"); - }); - setCursor($textarea[0], position +1); + var start = $textarea.prop('selectionStart'); + var end = $textarea.prop('selectionEnd'); + if (typeof start !== 'undefined') { + if (start === end) { + $textarea.val(function (i, val) { + return splice(val, start, "\t"); + }); + setCursor($textarea[0], start +1); + } else { + // indentation?? this ought to be fun. + + } } // prevent default behaviour for tab e.preventDefault(); From 284da6a4e9bb0c807473d8a434462e4eaf8a9a1a Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 13 Apr 2016 10:16:44 +0200 Subject: [PATCH 61/69] minor improvements I made to the /hack/ pad last night --- www/hack/main.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/www/hack/main.js b/www/hack/main.js index f5618c8b3..b775b73dc 100644 --- a/www/hack/main.js +++ b/www/hack/main.js @@ -58,7 +58,7 @@ define([ window.alert("Server Connection Lost"); }; - var rt = Realtime.start(config); + var rt = window.rt = Realtime.start(config); var cursor = Cursor($textarea[0]); @@ -114,13 +114,20 @@ define([ } } + // simulate a keypress so the event goes through.. // prevent default behaviour for tab e.preventDefault(); + rt.bumpSharejs(); + break; default: break; } }); + $textarea.on('change', function () { + rt.bumpSharejs(); + }); + $run.click(function (e) { e.preventDefault(); var content = $textarea.val(); From 8a369635820fe5a390336af8bff655504161a5cc Mon Sep 17 00:00:00 2001 From: Caleb James DeLisle Date: Wed, 13 Apr 2016 13:51:01 +0200 Subject: [PATCH 62/69] Enable ChainPad PARANOIA mode but remove the part which causes most slowness --- www/common/chainpad.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/www/common/chainpad.js b/www/common/chainpad.js index 9d9dcc071..134d80ce2 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -368,7 +368,7 @@ var random = Patch.random = function (doc, opCount) { * along with this program. If not, see . */ -var PARANOIA = module.exports.PARANOIA = false; +var PARANOIA = module.exports.PARANOIA = true; /* throw errors over non-compliant messages which would otherwise be treated as invalid */ var TESTING = module.exports.TESTING = true; @@ -832,7 +832,7 @@ var check = ChainPad.check = function(realtime) { Common.assert(uiDoc === realtime.userInterfaceContent); } - var doc = realtime.authDoc; + /*var doc = realtime.authDoc; var patchMsg = realtime.best; Common.assert(patchMsg.content.inverseOf.parentHash === realtime.uncommitted.parentHash); var patches = []; @@ -844,7 +844,7 @@ var check = ChainPad.check = function(realtime) { while ((patchMsg = patches.pop())) { doc = Patch.apply(patchMsg.content, doc); } - Common.assert(doc === realtime.authDoc); + Common.assert(doc === realtime.authDoc);*/ }; var doOperation = ChainPad.doOperation = function (realtime, op) { From 0537c289198f21627997efebe358224d20c6c940 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 13 Apr 2016 13:53:20 +0200 Subject: [PATCH 63/69] Add switchable logging to TextPatcher.js --- www/common/TextPatcher.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/www/common/TextPatcher.js b/www/common/TextPatcher.js index 66858826d..ee9c6fb61 100644 --- a/www/common/TextPatcher.js +++ b/www/common/TextPatcher.js @@ -79,14 +79,15 @@ var log = function (text, op) { Due to its reliance on patch, applyChange has side effects on the supplied realtime facade. */ -var applyChange = function(ctx, oldval, newval) { +var applyChange = function(ctx, oldval, newval, logging) { var op = diff(oldval, newval); - // log(oldval, op) + if (logging) { log(oldval, op) } patch(ctx, op); }; var create = function(config) { var ctx = config.realtime; + var logging = config.logging; // initial state will always fail the !== check in genop. // because nothing will equal this object @@ -100,7 +101,7 @@ var create = function(config) { // propogate() return function (newContent) { if (newContent !== content) { - applyChange(ctx, ctx.getUserDoc(), newContent); + applyChange(ctx, ctx.getUserDoc(), newContent, logging); if (ctx.getUserDoc() !== newContent) { console.log("Expected that: `ctx.getUserDoc() === newContent`!"); } From 259772dd62035c03759a075b2e30a41f98b08c7b Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 13 Apr 2016 13:54:25 +0200 Subject: [PATCH 64/69] Turn on TextPatcher logging for _socket --- www/common/RealtimeTextSocket.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/www/common/RealtimeTextSocket.js b/www/common/RealtimeTextSocket.js index edb8e3791..4821ce454 100644 --- a/www/common/RealtimeTextSocket.js +++ b/www/common/RealtimeTextSocket.js @@ -264,7 +264,8 @@ define([ }, 200); toReturn.patchText = TextPatcher.create({ - realtime: realtime + realtime: realtime, + logging: true }); realtime.start(); From f4c5b2a99646f9310c0b496d69907ceadb74b72f Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 13 Apr 2016 14:51:15 +0200 Subject: [PATCH 65/69] Add sane defaults to TextPatcher diffs --- www/common/TextPatcher.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/www/common/TextPatcher.js b/www/common/TextPatcher.js index ee9c6fb61..98ccea65a 100644 --- a/www/common/TextPatcher.js +++ b/www/common/TextPatcher.js @@ -23,8 +23,8 @@ var diff = function (oldval, newval) { commonEnd++; } - var toRemove; - var toInsert; + var toRemove = 0; + var toInsert = ''; /* throw some assertions in here before dropping patches into the realtime */ if (oldval.length !== commonStart + commonEnd) { From a1fe941f6990fe733a8ed0ca76c875c6bdb1e5d8 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 13 Apr 2016 15:00:19 +0200 Subject: [PATCH 66/69] Always serialize the DOM in one way. --- www/_socket/main.js | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/www/_socket/main.js b/www/_socket/main.js index 67bfaff89..042bd3c20 100644 --- a/www/_socket/main.js +++ b/www/_socket/main.js @@ -45,6 +45,16 @@ define([ return true; }; + /* catch `type="_moz"` before it goes over the wire */ + var brFilter = function (hj) { + if (hj[1].type === '_moz') { hj[1].type = undefined; } + return hj; + }; + + var stringifyDOM = function (dom) { + return JSON.stringify(Hyperjson.fromDOM(dom, isNotMagicLine, brFilter)); + }; + var andThen = function (Ckeditor) { $(window).on('hashchange', function() { window.location.reload(); @@ -76,6 +86,8 @@ define([ var inner = window.inner = documentBody; var cursor = window.cursor = Cursor(inner); + + var setEditable = function (bool) { // careful about putting attributes onto the DOM // they get put into the chain, and you can have trouble @@ -165,8 +177,7 @@ define([ doc: inner, // provide initialstate... - initialState: JSON.stringify(Hyperjson - .fromDOM(inner, isNotMagicLine)) || '{}', + initialState: stringifyDOM(inner) || '{}', // really basic operational transform // reject patch if it results in invalid JSON @@ -221,7 +232,8 @@ define([ // build a dom from HJSON, diff, and patch the editor applyHjson(shjson); - var shjson2 = JSON.stringify(Hyperjson.fromDOM(inner)); + var shjson2 = stringifyDOM(inner); + if (shjson2 !== shjson) { console.error("shjson2 !== shjson"); module.realtimeInput.patchText(shjson2); @@ -255,11 +267,6 @@ define([ var rti = module.realtimeInput = realtimeInput.start(realtimeOptions); - /* catch `type="_moz"` before it goes over the wire */ - var brFilter = function (hj) { - if (hj[1].type === '_moz') { hj[1].type = undefined; } - return hj; - }; /* It's incredibly important that you assign 'rti.onLocal' It's used inside of realtimeInput to make sure that all changes @@ -270,7 +277,7 @@ define([ the code less extensible. */ var propogate = rti.onLocal = function () { - var shjson = JSON.stringify(Hyperjson.fromDOM(inner, isNotMagicLine, brFilter)); + var shjson = stringifyDOM(inner); if (!rti.patchText(shjson)) { return; } From c867ab04ab8d31ffa5bf30e83b3c2da4a3ed1a85 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 13 Apr 2016 16:01:45 +0200 Subject: [PATCH 67/69] rename /_socket/ app to /p/ --- www/{_socket => p}/index.html | 0 www/{_socket => p}/inner.html | 0 www/{_socket => p}/main.js | 0 www/{_socket => p}/toolbar.js | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename www/{_socket => p}/index.html (100%) rename www/{_socket => p}/inner.html (100%) rename www/{_socket => p}/main.js (100%) rename www/{_socket => p}/toolbar.js (100%) diff --git a/www/_socket/index.html b/www/p/index.html similarity index 100% rename from www/_socket/index.html rename to www/p/index.html diff --git a/www/_socket/inner.html b/www/p/inner.html similarity index 100% rename from www/_socket/inner.html rename to www/p/inner.html diff --git a/www/_socket/main.js b/www/p/main.js similarity index 100% rename from www/_socket/main.js rename to www/p/main.js diff --git a/www/_socket/toolbar.js b/www/p/toolbar.js similarity index 100% rename from www/_socket/toolbar.js rename to www/p/toolbar.js From 6dabded135ab4a6ea54659710df17b9408870719 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 13 Apr 2016 16:21:17 +0200 Subject: [PATCH 68/69] correct path for having renamed _socket --- www/p/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/p/main.js b/www/p/main.js index 042bd3c20..9160cbd67 100644 --- a/www/p/main.js +++ b/www/p/main.js @@ -5,7 +5,7 @@ define([ '/common/RealtimeTextSocket.js', '/common/hyperjson.js', '/common/hyperscript.js', - '/_socket/toolbar.js', + '/p/toolbar.js', '/common/cursor.js', '/common/json-ot.js', '/common/TypingTests.js', From 2bab22cc1f9f307ebd411c73778b5146ff32151d Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 12 May 2016 14:59:45 +0200 Subject: [PATCH 69/69] remove note about jquery sheet --- customize.dist/index.html | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/customize.dist/index.html b/customize.dist/index.html index b652894e6..d5080fcc5 100644 --- a/customize.dist/index.html +++ b/customize.dist/index.html @@ -98,9 +98,8 @@

CryptPad is the zero knowledge realtime collaborative editor. Encryption carried out in your web browser protects the data from the server, the cloud and the NSA. This project uses the CKEditor Visual Editor - the ChainPad realtime engine and now - jQuery.sheet for realtime spreadsheet - editing! The secret encryption key is stored in the URL + the ChainPad realtime engine. The secret + encryption key is stored in the URL fragment identifier which is never sent to the server but is available to javascript so by sharing the URL, you give authorization to others who want to participate.