From dff8143faddf0bdcc19077915f8b4166c874bb47 Mon Sep 17 00:00:00 2001 From: Caleb James DeLisle Date: Thu, 11 Feb 2016 12:28:05 +0100 Subject: [PATCH 1/4] Fix the lvl storage --- package.json | 4 +- storage/lvl.js | 107 ++++++++++++++++++------------------------------- 2 files changed, 42 insertions(+), 69 deletions(-) diff --git a/package.json b/package.json index 1c1271df9..e62340d72 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,8 @@ "version": "0.1.0", "dependencies": { "express": "~4.10.1", - "ws": "^1.0.1" + "ws": "^1.0.1", + "level": "~1.4.0", + "nthen": "~0.1.0" } } diff --git a/storage/lvl.js b/storage/lvl.js index b9c1c5bcc..179d1fc73 100644 --- a/storage/lvl.js +++ b/storage/lvl.js @@ -1,77 +1,48 @@ -var level=require("level"); +var Level = require("level"); +var nThen = require('nthen'); -module.exports.create = function(conf,cb){ - console.log("Loading leveldb"); +var getIndex = function(db, cName, cb) { + db.get(cName+'=>index', function(e, out){ + if (e) { throw e; } + cb(parseInt(out)); + }); +}; - var db=level(conf.levelPath||'./test.level.db'), - indices={}, - Channel={}; +var insert = function (db, channelName, content, cb) { + var index; + nThen(function (waitFor) { + getIndex(db, channelName, waitFor(function (i) { index = i; })); + }).nThen(function (waitFor) { + db.put(channelName+'=>index', ''+index, waitFor(function (e) { if (e) { throw e; } })); + }).nThen(function (waitFor) { + db.put(channelName+'=>'+index, content, waitFor(function (e) { if (e) { throw e; } })); + }).nThen(cb); +}; - var makeChannel=function(cName){ - Channel[cName]={ - lastModified:0, - messages:[], +var getMessages = function (db, channelName, msgHandler) { + var index; + nThen(function (waitFor) { + getIndex(db, channelName, waitFor(function (i) { index = i; })); + }).nThen(function (waitFor) { + var again = function (i) { + db.get(channelName + '=>' + i, waitFor(function (e, out) { + if (e) { throw e; } + msgHandler(out); + if (i < index) { again(i+1); } + })); }; - }, - makeIndex=function(cName){ - // initializing to negative one means we can increment on inserts - // so we always start from zero. - indices[cName]=-1; - }, - loadIndex=function(cName, out){ - indices[cName]=parseInt(out); - typeof indices[cName] !== 'number' && - console.error("FOUND A NON-NUMERIC INDEX for channel: %s", cName); - }, - getIndex=function(cName,f){ - if(typeof indices[cName] !== 'undefined'){ - f(indices[cName]); - }else{ - // get and increment the channelIndex - db.get(cName+'=>index',function(e,out){ - if(e){ - // it doesn't exist, so initialize it - makeIndex(cName); - }else{ - // it exists. parse it - loadIndex(cName,out); - } - f(indices[cName]); - }); - } - }; + again(0); + }); +}; +module.exports.create = function (conf, cb) { + var db = Level(conf.levelPath || './test.level.db'); cb({ - message: function(cName,content,cb){ - getIndex(cName,function(index){ - var index = ++indices[cName]; - db.put(cName+'=>index', ''+index,function(e){ - if(e) console.error(e); - }); - db.put(cName+'=>'+index, content, function(err){ - if(err){ - console.log(err); - } - cb(); - }); - }); - }, - getMessages: function(cName, cb){ - /* get all messages relating to a channel */ - getIndex(cName, function(index){ - var last = index, - i = 0, - next = function () { - db.get(cName+'=>'+i, function (e,out) { - if(e) return console.error(e); - cb(out); - if (++i <= last) { - next(); - } - }); - }; - next(); - }); + message: function (channelName, content, cb) { + insert(db, channelName, content, cb); }, + getMessages: function (channelName, msgHandler) { + getMessages(db, channelName, msgHandler); + } }); }; From e16299b5c41325e75db471e0fae0fc5456b5949a Mon Sep 17 00:00:00 2001 From: Caleb James DeLisle Date: Thu, 11 Feb 2016 12:33:12 +0100 Subject: [PATCH 2/4] Revert "restub OT until it's verified" This reverts commit 350c9e6c7658d93df015fe2b0c82934e0d861bf6. --- www/vdom/main.js | 63 ++++++++++++++---------------------------------- 1 file changed, 18 insertions(+), 45 deletions(-) diff --git a/www/vdom/main.js b/www/vdom/main.js index 9be13486f..f8f13ce09 100644 --- a/www/vdom/main.js +++ b/www/vdom/main.js @@ -6,12 +6,9 @@ define([ '/common/convert.js', '/common/toolbar.js', '/common/cursor.js', - '/common/Operation.js', '/bower_components/jquery/dist/jquery.min.js', '/customize/pad.js' -], function (Config, Messages, Crypto, realtimeInput, Convert, Toolbar, Cursor, Operation) { - window.Operation = Operation; - +], function (Config, Messages, Crypto, realtimeInput, Convert, Toolbar, Cursor) { var $ = window.jQuery; var ifrw = $('#pad-iframe')[0].contentWindow; window.Ckeditor = ifrw.CKEDITOR; @@ -107,53 +104,29 @@ define([ onInit: onInit, transformFunction : function (text, toTransform, transformBy) { + /* FIXME + operational transform on json shouldn't be in all editors + just those transmitting/expecting JSON + */ + false && console.log({ + text: text, + toTransform: toTransform, + transformBy: transformBy + }); + // returning **null** breaks out of the loop // which transforms conflicting operations // in theory this should prevent us from producing bad JSON return null; - }, - - // OT + } /* - transformFunction: function (text, toTransform, transformBy) { - if (toTransform.offset > transformBy.offset) { - if (toTransform.offset > transformBy.offset + transformBy.toRemove) { - // simple rebase - toTransform.offset -= transformBy.toRemove; - toTransform.offset += transformBy.toInsert.length; - - // TODO check that this is using the correct parameters - // TODO get the actual library - // TODO functionize this because it's repeated - - var temp = Operation.apply(text, toTransform) - try { - JSON.parse(temp); - } catch (err) { - console.error(err.stack); - return null; - } - return toTransform; - } - // goto the end, anything you deleted that they also deleted should be skipped. - var newOffset = transformBy.offset + transformBy.toInsert.length; - toTransform.toRemove = 0; //-= (newOffset - toTransform.offset); - if (toTransform.toRemove < 0) { toTransform.toRemove = 0; } - toTransform.offset = newOffset; - if (toTransform.toInsert.length === 0 && toTransform.toRemove === 0) { - return null; - } - return toTransform; - } - if (toTransform.offset + toTransform.toRemove < transformBy.offset) { - return toTransform; - } - toTransform.toRemove = transformBy.offset - toTransform.offset; - if (toTransform.toInsert.length === 0 && toTransform.toRemove === 0) { - return null; + FIXME NOT A REAL FUNCTION WONT WORK + transformFunction: function (state0str, toTransform, transformBy) { + var state1A = JSON.parse(Operation.apply(state0str, transformBy)); + var state1B = JSON.parse(Operation.apply(state0str, toTransform)); + var state0 = JSON.parse(state0str); } - return toTransform; - } */ + */ }); $textarea.val(JSON.stringify(Convert.dom.to.hjson(inner))); From 37db31b09e9ec147385002f5b613168fc67f8e79 Mon Sep 17 00:00:00 2001 From: Caleb James DeLisle Date: Thu, 11 Feb 2016 12:34:06 +0100 Subject: [PATCH 3/4] Attempt an operational transformation and if the result is not valid JSON then fail --- www/common/chainpad.js | 1 + www/vdom/main.js | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/www/common/chainpad.js b/www/common/chainpad.js index 930ddb403..01aae63f9 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -1163,6 +1163,7 @@ module.exports.create = function (userName, authToken, channelId, initialState, Common.assert(typeof(initialState) === 'string'); var realtime = ChainPad.create(userName, authToken, channelId, initialState, conf); return { + Operation: Operation, onPatch: enterChainPad(realtime, function (handler) { Common.assert(typeof(handler) === 'function'); realtime.patchHandlers.push(handler); diff --git a/www/vdom/main.js b/www/vdom/main.js index f8f13ce09..b734039b1 100644 --- a/www/vdom/main.js +++ b/www/vdom/main.js @@ -114,6 +114,11 @@ define([ transformBy: transformBy }); + var resultOp = ChainPad.Operation.transform0(text, toTransform, transformBy); + var text2 = ChainPad.Operation.apply(transformBy, text); + var text3 = ChainPad.Operation.apply(resultOp, text2); + try { JSON.parse(text3); return resultOp; } catch (e) { console.log(e); } + // returning **null** breaks out of the loop // which transforms conflicting operations // in theory this should prevent us from producing bad JSON From 2031191261c1c7fd44393899a230a6fe54f77e58 Mon Sep 17 00:00:00 2001 From: Caleb James DeLisle Date: Thu, 11 Feb 2016 13:13:58 +0100 Subject: [PATCH 4/4] testing=false --- www/common/chainpad.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/www/common/chainpad.js b/www/common/chainpad.js index 01aae63f9..cf6193df8 100644 --- a/www/common/chainpad.js +++ b/www/common/chainpad.js @@ -372,7 +372,7 @@ var random = Patch.random = function (doc, opCount) { var PARANOIA = module.exports.PARANOIA = false; /* throw errors over non-compliant messages which would otherwise be treated as invalid */ -var TESTING = module.exports.TESTING = true; +var TESTING = module.exports.TESTING = false; var assert = module.exports.assert = function (expr) { if (!expr) { throw new Error("Failed assertion"); } @@ -1283,7 +1283,7 @@ var clone = Operation.clone = function (op) { /** * @param op the operation to apply. - * @param doc the content to apply the operation on + * @param doc the content to apply the operation on */ var apply = Operation.apply = function (op, doc) {