Merge branch 'master' into staging

lib/hk-util.js

@@ -407,19 +407,9 @@ const getHistoryOffset = (Env, channelName, lastKnownHash, _cb) => {
 
             // check if the "hash" the client is requesting exists in the index
             const lkh = index.offsetByHash[lastKnownHash];
-            // we evict old hashes from the index as new checkpoints are discovered.
+
-            // if someone connects and asks for a hash that is no longer relevant,
+            // fall through to the next block if the offset of the hash in question is not in memory
-            // we tell them it's an invalid request. This is because of the semantics of "GET_HISTORY"
+            if (lastKnownHash && typeof(lkh) !== "number") { return; }
-            // which is only ever used when connecting or reconnecting in typical uses of history...
-            // this assumption should hold for uses by chainpad, but perhaps not for other uses cases.
-            // EXCEPT: other cases don't use checkpoints!
-            // clients that are told that their request is invalid should just make another request
-            // without specifying the hash, and just trust the server to give them the relevant data.
-            // QUESTION: does this mean mailboxes are causing the server to store too much stuff in memory?
-            if (lastKnownHash && typeof(lkh) !== "number") {
-                waitFor.abort();
-                return void cb(new Error('EUNKNOWN'));
-            }
 
             // Since last 2 checkpoints
             if (!lastKnownHash) {
@@ -441,13 +431,13 @@ const getHistoryOffset = (Env, channelName, lastKnownHash, _cb) => {
             offset = lkh;
         }));
     }).nThen((w) => {
-        // if offset is less than zero then presumably the channel has no messages
+        // skip past this block if the offset is anything other than -1
-        // returning falls through to the next block and therefore returns -1
+        // this basically makes these first two nThen blocks behave like if-else
         if (offset !== -1) { return; }
 
-        // do a lookup from the index
+        // either the message exists in history but is not in the cached index
-        // FIXME maybe we don't need this anymore?
+        // or it does not exist at all. In either case 'getHashOffset' is expected
-        // otherwise we have a non-negative offset and we can start to read from there
+        // to return a number: -1 if not present, positive interger otherwise
         Env.getHashOffset(channelName, lastKnownHash, w(function (err, _offset) {
             if (err) {
                 w.abort();
@@ -482,7 +472,9 @@ const getHistoryAsync = (Env, channelName, lastKnownHash, beforeHash, handler, c
             offset = os;
         }));
     }).nThen((waitFor) => {
-        if (offset === -1) { return void cb(new Error("could not find offset")); }
+        if (offset === -1) {
+            return void cb(new Error('EUNKNOWN'));
+        }
         const start = (beforeHash) ? 0 : offset;
         store.readMessagesBin(channelName, start, (msgObj, readMore, abort) => {
             if (beforeHash && msgObj.offset >= offset) { return void abort(); }

lib/workers/db-worker.js

@@ -153,7 +153,8 @@ const computeIndex = function (data, cb) {
                     messageBuf = [];
                 }
             } else if (messageBuf.length > 100 && cpIndex.length === 0) {
-                messageBuf = messageBuf.slice(0, 50);
+                // take the last 50 messages
+                messageBuf = messageBuf.slice(-50);
             }
             // if it's not metadata or a checkpoint then it should be a regular message
             // store it in the buffer
@@ -352,7 +353,8 @@ const getMultipleFileSize = function (data, cb) {
 
 const getHashOffset = function (data, cb) {
     const channelName = data.channel;
-    const lastKnownHash = data.lastKnownHash;
+    const lastKnownHash = data.hash;
+    if (typeof(lastKnownHash) !== 'string') { return void cb("INVALID_HASH"); }
 
     var offset = -1;
     store.readMessagesBin(channelName, 0, (msgObj, readMore, abort) => {

scripts/tests/test-mailbox.js

@@ -11,8 +11,14 @@ var Hash = require("../../www/common/common-hash");
 var CpNetflux = require("../../www/bower_components/chainpad-netflux");
 var Util = require("../../lib/common-util");
 
-var createMailbox = function (config, cb) {
+// you need more than 100 messages in the history, and you need a lastKnownHash between "50" and "length - 50"
+
+var createMailbox = function (config, _cb) {
+    var cb = Util.once(Util.mkAsync(_cb));
+
     var webchannel;
+    var user = config.user;
+    user.messages = [];
 
     CpNetflux.start({
         network: config.network,
@@ -21,11 +27,16 @@ var createMailbox = function (config, cb) {
         owners: [ config.edPublic ],
 
         noChainPad: true,
+
+        lastKnownHash: config.lastKnownHash,
+        onChannelError: function (err) {
+            cb(err);
+        },
         onConnect: function (wc /*, sendMessage */) {
             webchannel = wc;
         },
-        onMessage: function (/* msg, user, vKey, isCp, hash, author */) {
+        onMessage: function (msg /*, user, vKey, isCp, hash, author */) {
-
+            user.messages.push(msg);
         },
         onReady: function () {
             cb(void 0, webchannel);
@@ -37,6 +48,8 @@ process.on('unhandledRejection', function (err) {
     console.error(err);
 });
 
+var state = {};
+
 var makeCurveKeys = function () {
     var pair = Nacl.box.keyPair();
     return {
@@ -53,6 +66,10 @@ var makeEdKeys = function () {
     };
 };
 
+var edKeys = makeEdKeys();
+var curveKeys = makeCurveKeys();
+var mailboxChannel = Hash.createChannelId();
+
 var createUser = function (config, cb) {
     // config should contain keys for a team rpc (ed)
         // teamEdKeys
@@ -75,11 +92,11 @@ var createUser = function (config, cb) {
         // make all the parameters you'll need
 
         var network = user.network = user.config.network;
-        user.edKeys = makeEdKeys();
+        user.edKeys = edKeys;
+        user.curveKeys = curveKeys;
 
-        user.curveKeys = makeCurveKeys();
         user.mailbox = Mailbox.createEncryptor(user.curveKeys);
-        user.mailboxChannel = Hash.createChannelId();
+        user.mailboxChannel = mailboxChannel;
 
         // create an anon rpc for alice
         Rpc.createAnonymous(network, w(function (err, rpc) {
@@ -109,6 +126,11 @@ var createUser = function (config, cb) {
     }).nThen(function (w) {
         // create and subscribe to your mailbox
         createMailbox({
+            user: user,
+
+
+            lastKnownHash: config.lastKnownHash,
+
             network: user.network,
             channel: user.mailboxChannel,
             crypto: user.mailbox,
@@ -116,8 +138,9 @@ var createUser = function (config, cb) {
         }, w(function (err /*, wc*/) {
             if (err) {
                 w.abort();
-                console.error("Mailbox creation error");
+                //console.error("Mailbox creation error");
-                process.exit(1);
+                cb(err);
+                //process.exit(1);
             }
             //wc.leave();
         }));
@@ -135,14 +158,10 @@ var createUser = function (config, cb) {
 
 var alice;
 
-var sharedConfig = {
-    teamEdKeys: makeEdKeys(),
-    teamCurveKeys: makeCurveKeys(),
-    rosterSeed: Crypto.Team.createSeed(),
-};
-
 nThen(function  (w) {
-    createUser(sharedConfig, w(function (err, _alice) {
+    createUser({
+        //sharedConfig
+    }, w(function (err, _alice) {
         if (err) {
             w.abort();
             return void console.log(err);
@@ -163,13 +182,18 @@ nThen(function  (w) {
     var i = 0;
     var next = w();
 
+    state.hashes = [];
+
     var send = function () {
-        if (i++ >= 300) { return next(); }
+        if (i++ >= 160) { return next(); }
 
         var msg = alice.mailbox.encrypt(JSON.stringify({
             pewpew: 'bangbang',
         }), alice.curveKeys.curvePublic);
 
+        var hash = msg.slice(0, 64);
+        state.hashes.push(hash);
+
         alice.anonRpc.send('WRITE_PRIVATE_MESSAGE', [
             alice.mailboxChannel,
             msg
@@ -177,10 +201,33 @@ nThen(function  (w) {
         ], w(function (err) {
             if (err) { throw new Error(err); }
             console.log('message %s written successfully', i);
-            setTimeout(send, 250);
+            setTimeout(send, 15);
         }));
     };
     send();
+}).nThen(function (w) {
+    console.log("Connecting with second user");
+    createUser({
+        lastKnownHash: state.hashes[55],
+    }, w(function (err, _alice) {
+        if (err) {
+            w.abort();
+            console.log("lastKnownHash: ", state.hashes[55]);
+            console.log(err);
+            process.exit(1);
+            //return void console.log(err);
+        }
+        var user = state.alice2 = _alice;
+
+        if (user.messages.length === 105) {
+            process.exit(0);
+        }
+        //console.log(user.messages, user.messages.length);
+        process.exit(1);
+    }));
+}).nThen(function () {
+    
+
 }).nThen(function () {
     alice.cleanup();
     //bob.cleanup();