From 9e1f49c1779d1e66b3a9c09a6e54fd0acd42a827 Mon Sep 17 00:00:00 2001
From: yflory <yann.flory@xwiki.com>
Date: Wed, 15 Jan 2020 15:33:52 +0100
Subject: [PATCH 1/6] Enable spreadsheets for anonymous users

---
 www/common/application_config_internal.js | 2 +-
 www/common/onlyoffice/inner.js            | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/www/common/application_config_internal.js b/www/common/application_config_internal.js
index d4c1954e4..900a9ec28 100644
--- a/www/common/application_config_internal.js
+++ b/www/common/application_config_internal.js
@@ -20,7 +20,7 @@ define(function() {
      * users and these users will be redirected to the login page if they still try to access
      * the app
      */
-    config.registeredOnlyTypes = ['file', 'contacts', 'oodoc', 'ooslide', 'sheet', 'notifications'];
+    config.registeredOnlyTypes = ['file', 'contacts', 'oodoc', 'ooslide', 'notifications'];
 
     /* CryptPad is available is multiple languages, but only English and French are maintained
      * by the developers. The other languages may be outdated, and any missing string for a langauge
diff --git a/www/common/onlyoffice/inner.js b/www/common/onlyoffice/inner.js
index a40ccccae..c22ca019c 100644
--- a/www/common/onlyoffice/inner.js
+++ b/www/common/onlyoffice/inner.js
@@ -584,7 +584,9 @@ define([
                 ooChannel.cpIndex++;
                 ooChannel.lastHash = hash;
                 // Check if a checkpoint is needed
-                if (ooChannel.cpIndex % CHECKPOINT_INTERVAL === 0) {
+                var lastCp = getLastCp();
+                if (common.isLoggedIn() && (ooChannel.cpIndex % CHECKPOINT_INTERVAL === 0 ||
+                            (ooChannel.cpIndex - lastCp.index) > CHECKPOINT_INTERVAL)) {
                     makeCheckpoint();
                 }
                 // Remove my lock
@@ -662,7 +664,7 @@ define([
         var startOO = function (blob, file) {
             if (APP.ooconfig) { return void console.error('already started'); }
             var url = URL.createObjectURL(blob);
-            var lock = readOnly || !common.isLoggedIn();
+            var lock = readOnly;// || !common.isLoggedIn();
 
             // Config
             APP.ooconfig = {

From b80d3cf24f232f33e448e849df9daa444b07f832 Mon Sep 17 00:00:00 2001
From: yflory <yann.flory@xwiki.com>
Date: Thu, 16 Jan 2020 14:52:04 +0100
Subject: [PATCH 2/6] Ability to provide a dropdown container to a text input.
 Applied to framework export.

---
 .../src/less2/include/alertify.less           |  3 ++
 customize.dist/src/less2/include/buttons.less | 17 ++++++++++
 .../src/less2/include/dropdown.less           |  3 +-
 www/common/common-interface.js                | 15 +++++++-
 www/common/common-ui-elements.js              | 11 ++++--
 www/common/sframe-app-framework.js            | 34 ++++++++++++++++++-
 6 files changed, 76 insertions(+), 7 deletions(-)

diff --git a/customize.dist/src/less2/include/alertify.less b/customize.dist/src/less2/include/alertify.less
index a2b787e1c..83c9068eb 100644
--- a/customize.dist/src/less2/include/alertify.less
+++ b/customize.dist/src/less2/include/alertify.less
@@ -168,6 +168,9 @@
                     margin-bottom: 0;
                 }
             }
+            .cp-alertify-type-container {
+                overflow: visible !important;
+            }
             .alertify-tabs {
                 max-height: 100%;
                 display: flex;
diff --git a/customize.dist/src/less2/include/buttons.less b/customize.dist/src/less2/include/buttons.less
index fb99471b6..ad6aaf9cc 100644
--- a/customize.dist/src/less2/include/buttons.less
+++ b/customize.dist/src/less2/include/buttons.less
@@ -23,6 +23,23 @@
         }
     }
 
+    div.cp-alertify-type {
+        display: flex;
+        input {
+            margin: 0;
+            flex: 1;
+            min-width: 0;
+        }
+        span {
+            button {
+                margin: 0;
+                height: 100%;
+                margin-left: -1px;
+                text-transform: unset !important;
+            }
+        }
+    }
+
     textarea {
         padding: 8px;
         &[readonly] {
diff --git a/customize.dist/src/less2/include/dropdown.less b/customize.dist/src/less2/include/dropdown.less
index 10044528e..271603216 100644
--- a/customize.dist/src/less2/include/dropdown.less
+++ b/customize.dist/src/less2/include/dropdown.less
@@ -17,8 +17,7 @@
 
         button {
             .fa-caret-down {
-                margin-right: 0px;
-                margin-left: 5px;
+                margin-right: 1em !important;
             }
             * {
                 .tools_unselectable();
diff --git a/www/common/common-interface.js b/www/common/common-interface.js
index 55d7e6939..d4b3059b4 100644
--- a/www/common/common-interface.js
+++ b/www/common/common-interface.js
@@ -161,6 +161,17 @@ define([
         return h('p.msg', h('input', attrs));
     };
 
+    dialog.textTypeInput = function (dropdown) {
+        var attrs = {
+            type: 'text',
+            'class': 'cp-text-type-input',
+        };
+        return h('p.msg.cp-alertify-type-container', h('div.cp-alertify-type', [
+            h('input', attrs),
+            dropdown // must be a "span"
+        ]));
+    };
+
     dialog.nav = function (content) {
         return h('nav', content || [
             dialog.cancelButton(),
@@ -186,6 +197,7 @@ define([
             });
         };
         return $frame.click(function (e) {
+            $frame.find('.cp-dropdown-content').hide();
             e.stopPropagation();
         })[0];
     };
@@ -480,7 +492,8 @@ define([
         cb = cb || function () {};
         opt = opt || {};
 
-        var inputBlock = opt.password ? UI.passwordInput() : dialog.textInput();
+        var inputBlock = opt.password ? UI.passwordInput() :
+                            (opt.typeInput ? dialog.textTypeInput(opt.typeInput) : dialog.textInput());
         var input = $(inputBlock).is('input') ? inputBlock : $(inputBlock).find('input')[0];
         input.value = typeof(def) === 'string'? def: '';
 
diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js
index a195a1ba9..eb032bfaf 100644
--- a/www/common/common-ui-elements.js
+++ b/www/common/common-ui-elements.js
@@ -2896,9 +2896,11 @@ define([
         var $button = $('<button>', {
             'class': ''
         }).append($('<span>', {'class': 'cp-dropdown-button-title'}).html(config.text || ""));
-        /*$('<span>', {
-            'class': 'fa fa-caret-down',
-        }).appendTo($button);*/
+        if (config.caretDown) {
+            $('<span>', {
+                'class': 'fa fa-caret-down',
+            }).prependTo($button);
+        }
 
         // Menu
         var $innerblock = $('<div>', {'class': 'cp-dropdown-content'});
@@ -2965,6 +2967,9 @@ define([
         if (config.isSelect) {
             var pressed = '';
             var to;
+            $container.on('click', 'a', function () {
+                value = $(this).data('value');
+            });
             $container.keydown(function (e) {
                 var $value = $innerblock.find('[data-value].cp-dropdown-element-active:visible');
                 if (e.which === 38) { // Up
diff --git a/www/common/sframe-app-framework.js b/www/common/sframe-app-framework.js
index 84e73db01..ed8652809 100644
--- a/www/common/sframe-app-framework.js
+++ b/www/common/sframe-app-framework.js
@@ -6,9 +6,11 @@ define([
     '/bower_components/nthen/index.js',
     '/common/sframe-common.js',
     '/customize/messages.js',
+    '/common/hyperscript.js',
     '/common/common-util.js',
     '/common/common-hash.js',
     '/common/common-interface.js',
+    '/common/common-ui-elements.js',
     '/common/common-thumbnail.js',
     '/common/common-feedback.js',
     '/customize/application_config.js',
@@ -26,9 +28,11 @@ define([
     nThen,
     SFCommon,
     Messages,
+    h,
     Util,
     Hash,
     UI,
+    UIElements,
     Thumb,
     Feedback,
     AppConfig,
@@ -408,10 +412,36 @@ define([
             var $export = common.createButton('export', true, {}, function () {
                 var ext = (typeof(extension) === 'function') ? extension() : extension;
                 var suggestion = title.suggestTitle('cryptpad-document');
+                ext = ext || '.txt';
+                var types = [{
+                    tag: 'a',
+                    attributes: {
+                        'data-value': ext,
+                        'href': '#'
+                    },
+                    content: ext
+                }, {
+                    tag: 'a',
+                    attributes: {
+                        'data-value': '',
+                        'href': '#'
+                    },
+                    content: '&nbsp;'
+                }];
+                var dropdownConfig = {
+                    text: ext, // Button initial text
+                    caretDown: true,
+                    options: types, // Entries displayed in the menu
+                    isSelect: true,
+                    initialValue: ext,
+                    common: common
+                };
+                var $select = UIElements.createDropdown(dropdownConfig);
                 UI.prompt(Messages.exportPrompt,
-                    Util.fixFileName(suggestion) + ext, function (filename)
+                    Util.fixFileName(suggestion), function (filename)
                 {
                     if (!(typeof(filename) === 'string' && filename)) { return; }
+                    filename = filename + $select.getValue();
                     if (async) {
                         fe(function (blob) {
                             SaveAs(blob, filename);
@@ -420,6 +450,8 @@ define([
                     }
                     var blob = fe();
                     SaveAs(blob, filename);
+                }, {
+                    typeInput: $select[0]
                 });
             });
             toolbar.$drawer.append($export);

From cb6f0513a2ed590663daa3d712b72d78a0a5ed08 Mon Sep 17 00:00:00 2001
From: yflory <yann.flory@xwiki.com>
Date: Thu, 16 Jan 2020 17:51:02 +0100
Subject: [PATCH 3/6] Fix extension selector in pad export

---
 www/common/common-ui-elements.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js
index eb032bfaf..213887265 100644
--- a/www/common/common-ui-elements.js
+++ b/www/common/common-ui-elements.js
@@ -2969,6 +2969,9 @@ define([
             var to;
             $container.on('click', 'a', function () {
                 value = $(this).data('value');
+                var $val = $(this);
+                var textValue = $val.html() || value;
+                $button.find('.cp-dropdown-button-title').html(textValue);
             });
             $container.keydown(function (e) {
                 var $value = $innerblock.find('[data-value].cp-dropdown-element-active:visible');

From d1b8d8668e3b634b87a7f6f0bfe2f4c2a88e5555 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Benqu=C3=A9?= <mail@davidbenque.com>
Date: Thu, 16 Jan 2020 16:59:31 +0000
Subject: [PATCH 4/6] linting

---
 customize.dist/src/less2/include/modals-ui-elements.less | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/customize.dist/src/less2/include/modals-ui-elements.less b/customize.dist/src/less2/include/modals-ui-elements.less
index cee774a98..3e6855fbe 100644
--- a/customize.dist/src/less2/include/modals-ui-elements.less
+++ b/customize.dist/src/less2/include/modals-ui-elements.less
@@ -14,7 +14,7 @@
         .radio-group {
             display: flex;
             flex-direction: row;
-            &:not(:last-child){
+            &:not(:last-child) {
                 margin-bottom: 8px;
             }
             .cp-radio {

From 9de073c269ac8d7ac6f2f45641422448023b5a0f Mon Sep 17 00:00:00 2001
From: ansuz <ansuz@transitiontech.ca>
Date: Thu, 16 Jan 2020 14:25:00 -0500
Subject: [PATCH 5/6] finally get around to reorganizing the messiest part of
 history keeper

---
 historyKeeper.js | 416 ++++++++++++++++++++++++-----------------------
 1 file changed, 213 insertions(+), 203 deletions(-)

diff --git a/historyKeeper.js b/historyKeeper.js
index 59a694e8c..1035681d8 100644
--- a/historyKeeper.js
+++ b/historyKeeper.js
@@ -730,227 +730,204 @@ module.exports.create = function (cfg) {
         }
     };
 
-    /*  onDirectMessage
-        * exported for use by the netflux-server
-        * parses and handles all direct messages directed to the history keeper
-          * check if it's expired and execute all the associated side-effects
-          * routes queries to the appropriate handlers
-            * GET_HISTORY
-            * GET_HISTORY_RANGE
-            * GET_FULL_HISTORY
-            * RPC
-              * if the rpc has special hooks that the history keeper needs to be aware of...
-                * execute them here...
-
-    */
-    const onDirectMessage = function (ctx, seq, user, json) {
-        let parsed;
-        let channelName;
-
-        Log.silly('HK_MESSAGE', json);
-
-        try {
-            parsed = JSON.parse(json[2]);
-        } catch (err) {
-            Log.error("HK_PARSE_CLIENT_MESSAGE", json);
-            return;
-        }
-
-        // If the requested history is for an expired channel, abort
-        // Note the if we don't have the keys for that channel in metadata_cache, we'll
-        // have to abort later (once we know the expiration time)
-        if (checkExpired(ctx, parsed[1])) { return; }
-
-        if (parsed[0] === 'GET_HISTORY') {
-            // parsed[1] is the channel id
-            // parsed[2] is a validation key or an object containing metadata (optionnal)
-            // parsed[3] is the last known hash (optionnal)
-            sendMsg(ctx, user, [seq, 'ACK']);
-            channelName = parsed[1];
-            var config = parsed[2];
-            var metadata = {};
-            var lastKnownHash;
-
-            // clients can optionally pass a map of attributes
-            // if the channel already exists this map will be ignored
-            // otherwise it will be stored as the initial metadata state for the channel
-            if (config && typeof config === "object" && !Array.isArray(parsed[2])) {
-                lastKnownHash = config.lastKnownHash;
-                metadata = config.metadata || {};
-                if (metadata.expire) {
-                    metadata.expire = +metadata.expire * 1000 + (+new Date());
-                }
-            }
-            metadata.channel = channelName;
-            metadata.created = +new Date();
-
-            // if the user sends us an invalid key, we won't be able to validate their messages
-            // so they'll never get written to the log anyway. Let's just drop their message
-            // on the floor instead of doing a bunch of extra work
-            // TODO send them an error message so they know something is wrong
-            if (metadata.validateKey && !isValidValidateKeyString(metadata.validateKey)) {
-                return void Log.error('HK_INVALID_KEY', metadata.validateKey);
+    const handleGetHistory = function (ctx, seq, user, parsed) {
+        // parsed[1] is the channel id
+        // parsed[2] is a validation key or an object containing metadata (optionnal)
+        // parsed[3] is the last known hash (optionnal)
+        sendMsg(ctx, user, [seq, 'ACK']);
+        var channelName = parsed[1];
+        var config = parsed[2];
+        var metadata = {};
+        var lastKnownHash;
+
+        // clients can optionally pass a map of attributes
+        // if the channel already exists this map will be ignored
+        // otherwise it will be stored as the initial metadata state for the channel
+        if (config && typeof config === "object" && !Array.isArray(parsed[2])) {
+            lastKnownHash = config.lastKnownHash;
+            metadata = config.metadata || {};
+            if (metadata.expire) {
+                metadata.expire = +metadata.expire * 1000 + (+new Date());
             }
+        }
+        metadata.channel = channelName;
+        metadata.created = +new Date();
+
+        // if the user sends us an invalid key, we won't be able to validate their messages
+        // so they'll never get written to the log anyway. Let's just drop their message
+        // on the floor instead of doing a bunch of extra work
+        // TODO send them an error message so they know something is wrong
+        if (metadata.validateKey && !isValidValidateKeyString(metadata.validateKey)) {
+            return void Log.error('HK_INVALID_KEY', metadata.validateKey);
+        }
 
-            nThen(function (waitFor) {
-                var w = waitFor();
+        nThen(function (waitFor) {
+            var w = waitFor();
 
-                /*  unless this is a young channel, we will serve all messages from an offset
-                    this will not include the channel metadata, so we need to explicitly fetch that.
-                    unfortunately, we can't just serve it blindly, since then young channels will
-                    send the metadata twice, so let's do a quick check of what we're going to serve...
+            /*  unless this is a young channel, we will serve all messages from an offset
+                this will not include the channel metadata, so we need to explicitly fetch that.
+                unfortunately, we can't just serve it blindly, since then young channels will
+                send the metadata twice, so let's do a quick check of what we're going to serve...
+            */
+            getIndex(ctx, channelName, waitFor((err, index) => {
+                /*  if there's an error here, it should be encountered
+                    and handled by the next nThen block.
+                    so, let's just fall through...
                 */
-                getIndex(ctx, channelName, waitFor((err, index) => {
-                    /*  if there's an error here, it should be encountered
-                        and handled by the next nThen block.
-                        so, let's just fall through...
-                    */
-                    if (err) { return w(); }
-
-
-                    // it's possible that the channel doesn't have metadata
-                    // but in that case there's no point in checking if the channel expired
-                    // or in trying to send metadata, so just skip this block
-                    if (!index || !index.metadata) { return void w(); }
-                    // And then check if the channel is expired. If it is, send the error and abort
-                    // FIXME this is hard to read because 'checkExpired' has side effects
-                    if (checkExpired(ctx, channelName)) { return void waitFor.abort(); }
-                    // always send metadata with GET_HISTORY requests
-                    sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w);
-                }));
-            }).nThen(() => {
-                let msgCount = 0;
-
-                // TODO compute lastKnownHash in a manner such that it will always skip past the metadata line?
-                getHistoryAsync(ctx, channelName, lastKnownHash, false, (msg, readMore) => {
-                    if (!msg) { return; }
-                    msgCount++;
-                    // avoid sending the metadata message a second time
-                    if (isMetadataMessage(msg) && metadata_cache[channelName]) { return readMore(); }
-                    sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)], readMore);
-                }, (err) => {
-                    if (err && err.code !== 'ENOENT') {
-                        if (err.message !== 'EINVAL') { Log.error("HK_GET_HISTORY", err); }
-                        const parsedMsg = {error:err.message, channel: channelName};
-                        sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]);
-                        return;
-                    }
-
-                    const chan = ctx.channels[channelName];
+                if (err) { return w(); }
+
+
+                // it's possible that the channel doesn't have metadata
+                // but in that case there's no point in checking if the channel expired
+                // or in trying to send metadata, so just skip this block
+                if (!index || !index.metadata) { return void w(); }
+                // And then check if the channel is expired. If it is, send the error and abort
+                // FIXME this is hard to read because 'checkExpired' has side effects
+                if (checkExpired(ctx, channelName)) { return void waitFor.abort(); }
+                // always send metadata with GET_HISTORY requests
+                sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w);
+            }));
+        }).nThen(() => {
+            let msgCount = 0;
 
-                    if (msgCount === 0 && !metadata_cache[channelName] && chan && chan.indexOf(user) > -1) {
-                        metadata_cache[channelName] = metadata;
+            // TODO compute lastKnownHash in a manner such that it will always skip past the metadata line?
+            getHistoryAsync(ctx, channelName, lastKnownHash, false, (msg, readMore) => {
+                if (!msg) { return; }
+                msgCount++;
+                // avoid sending the metadata message a second time
+                if (isMetadataMessage(msg) && metadata_cache[channelName]) { return readMore(); }
+                sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(msg)], readMore);
+            }, (err) => {
+                if (err && err.code !== 'ENOENT') {
+                    if (err.message !== 'EINVAL') { Log.error("HK_GET_HISTORY", err); }
+                    const parsedMsg = {error:err.message, channel: channelName};
+                    sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]);
+                    return;
+                }
 
-                        // the index will have already been constructed and cached at this point
-                        // but it will not have detected any metadata because it hasn't been written yet
-                        // this means that the cache starts off as invalid, so we have to correct it
-                        if (chan && chan.index) { chan.index.metadata = metadata; }
+                const chan = ctx.channels[channelName];
+
+                if (msgCount === 0 && !metadata_cache[channelName] && chan && chan.indexOf(user) > -1) {
+                    metadata_cache[channelName] = metadata;
+
+                    // the index will have already been constructed and cached at this point
+                    // but it will not have detected any metadata because it hasn't been written yet
+                    // this means that the cache starts off as invalid, so we have to correct it
+                    if (chan && chan.index) { chan.index.metadata = metadata; }
+
+                    // new channels will always have their metadata written to a dedicated metadata log
+                    // but any lines after the first which are not amendments in a particular format will be ignored.
+                    // Thus we should be safe from race conditions here if just write metadata to the log as below...
+                    // TODO validate this logic
+                    // otherwise maybe we need to check that the metadata log is empty as well
+                    store.writeMetadata(channelName, JSON.stringify(metadata), function (err) {
+                        if (err) {
+                            // FIXME tell the user that there was a channel error?
+                            return void Log.error('HK_WRITE_METADATA', {
+                                channel: channelName,
+                                error: err,
+                            });
+                        }
+                    });
 
-                        // new channels will always have their metadata written to a dedicated metadata log
-                        // but any lines after the first which are not amendments in a particular format will be ignored.
-                        // Thus we should be safe from race conditions here if just write metadata to the log as below...
-                        // TODO validate this logic
-                        // otherwise maybe we need to check that the metadata log is empty as well
-                        store.writeMetadata(channelName, JSON.stringify(metadata), function (err) {
+                    // write tasks
+                    if(tasks && metadata.expire && typeof(metadata.expire) === 'number') {
+                        // the fun part...
+                        // the user has said they want this pad to expire at some point
+                        tasks.write(metadata.expire, "EXPIRE", [ channelName ], function (err) {
                             if (err) {
-                                // FIXME tell the user that there was a channel error?
-                                return void Log.error('HK_WRITE_METADATA', {
-                                    channel: channelName,
-                                    error: err,
-                                });
+                                // if there is an error, we don't want to crash the whole server...
+                                // just log it, and if there's a problem you'll be able to fix it
+                                // at a later date with the provided information
+                                Log.error('HK_CREATE_EXPIRE_TASK', err);
+                                Log.info('HK_INVALID_EXPIRE_TASK', JSON.stringify([metadata.expire, 'EXPIRE', channelName]));
                             }
                         });
-
-                        // write tasks
-                        if(tasks && metadata.expire && typeof(metadata.expire) === 'number') {
-                            // the fun part...
-                            // the user has said they want this pad to expire at some point
-                            tasks.write(metadata.expire, "EXPIRE", [ channelName ], function (err) {
-                                if (err) {
-                                    // if there is an error, we don't want to crash the whole server...
-                                    // just log it, and if there's a problem you'll be able to fix it
-                                    // at a later date with the provided information
-                                    Log.error('HK_CREATE_EXPIRE_TASK', err);
-                                    Log.info('HK_INVALID_EXPIRE_TASK', JSON.stringify([metadata.expire, 'EXPIRE', channelName]));
-                                }
-                            });
-                        }
-                        sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(metadata)]);
                     }
+                    sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(metadata)]);
+                }
 
-                    // End of history message:
-                    let parsedMsg = {state: 1, channel: channelName};
-                    sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]);
-                });
+                // End of history message:
+                let parsedMsg = {state: 1, channel: channelName};
+                sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]);
             });
-        } else if (parsed[0] === 'GET_HISTORY_RANGE') {
-            channelName = parsed[1];
-            var map = parsed[2];
-            if (!(map && typeof(map) === 'object')) {
-                return void sendMsg(ctx, user, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]);
-            }
+        });
+    };
 
-            var oldestKnownHash = map.from;
-            var desiredMessages = map.count;
-            var desiredCheckpoint = map.cpCount;
-            var txid = map.txid;
-            if (typeof(desiredMessages) !== 'number' && typeof(desiredCheckpoint) !== 'number') {
-                return void sendMsg(ctx, user, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]);
-            }
+    const handleGetHistoryRange = function (ctx, seq, user, parsed) {
+        var channelName = parsed[1];
+        var map = parsed[2];
+        if (!(map && typeof(map) === 'object')) {
+            return void sendMsg(ctx, user, [seq, 'ERROR', 'INVALID_ARGS', HISTORY_KEEPER_ID]);
+        }
 
-            if (!txid) {
-                return void sendMsg(ctx, user, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]);
-            }
+        var oldestKnownHash = map.from;
+        var desiredMessages = map.count;
+        var desiredCheckpoint = map.cpCount;
+        var txid = map.txid;
+        if (typeof(desiredMessages) !== 'number' && typeof(desiredCheckpoint) !== 'number') {
+            return void sendMsg(ctx, user, [seq, 'ERROR', 'UNSPECIFIED_COUNT', HISTORY_KEEPER_ID]);
+        }
 
-            sendMsg(ctx, user, [seq, 'ACK']);
-            return void getOlderHistory(channelName, oldestKnownHash, function (messages) {
-                var toSend = [];
-                if (typeof (desiredMessages) === "number") {
-                    toSend = messages.slice(-desiredMessages);
-                } else {
-                    let cpCount = 0;
-                    for (var i = messages.length - 1; i >= 0; i--) {
-                        if (/^cp\|/.test(messages[i][4]) && i !== (messages.length - 1)) {
-                            cpCount++;
-                        }
-                        toSend.unshift(messages[i]);
-                        if (cpCount >= desiredCheckpoint) { break; }
+        if (!txid) {
+            return void sendMsg(ctx, user, [seq, 'ERROR', 'NO_TXID', HISTORY_KEEPER_ID]);
+        }
+
+        sendMsg(ctx, user, [seq, 'ACK']);
+        return void getOlderHistory(channelName, oldestKnownHash, function (messages) {
+            var toSend = [];
+            if (typeof (desiredMessages) === "number") {
+                toSend = messages.slice(-desiredMessages);
+            } else {
+                let cpCount = 0;
+                for (var i = messages.length - 1; i >= 0; i--) {
+                    if (/^cp\|/.test(messages[i][4]) && i !== (messages.length - 1)) {
+                        cpCount++;
                     }
+                    toSend.unshift(messages[i]);
+                    if (cpCount >= desiredCheckpoint) { break; }
                 }
-                toSend.forEach(function (msg) {
-                    sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id,
-                        JSON.stringify(['HISTORY_RANGE', txid, msg])]);
-                });
-
+            }
+            toSend.forEach(function (msg) {
                 sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id,
-                    JSON.stringify(['HISTORY_RANGE_END', txid, channelName])
-                ]);
-            });
-        } else if (parsed[0] === 'GET_FULL_HISTORY') {
-            // parsed[1] is the channel id
-            // parsed[2] is a validation key (optionnal)
-            // parsed[3] is the last known hash (optionnal)
-            sendMsg(ctx, user, [seq, 'ACK']);
-
-            // FIXME should we send metadata here too?
-            // none of the clientside code which uses this API needs metadata, but it won't hurt to send it (2019-08-22)
-            getHistoryAsync(ctx, parsed[1], -1, false, (msg, readMore) => {
-                if (!msg) { return; }
-                sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['FULL_HISTORY', msg])], readMore);
-            }, (err) => {
-                let parsedMsg = ['FULL_HISTORY_END', parsed[1]];
-                if (err) {
-                    Log.error('HK_GET_FULL_HISTORY', err.stack);
-                    parsedMsg = ['ERROR', parsed[1], err.message];
-                }
-                sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]);
+                    JSON.stringify(['HISTORY_RANGE', txid, msg])]);
             });
-        } else if (rpc) {
-            /* RPC Calls...  */
-            var rpc_call = parsed.slice(1);
 
-            sendMsg(ctx, user, [seq, 'ACK']);
-            try {
+            sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id,
+                JSON.stringify(['HISTORY_RANGE_END', txid, channelName])
+            ]);
+        });
+    };
+
+    const handleGetFullHistory = function (ctx, seq, user, parsed) {
+        // parsed[1] is the channel id
+        // parsed[2] is a validation key (optionnal)
+        // parsed[3] is the last known hash (optionnal)
+        sendMsg(ctx, user, [seq, 'ACK']);
+
+        // FIXME should we send metadata here too?
+        // none of the clientside code which uses this API needs metadata, but it won't hurt to send it (2019-08-22)
+        return void getHistoryAsync(ctx, parsed[1], -1, false, (msg, readMore) => {
+            if (!msg) { return; }
+            sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(['FULL_HISTORY', msg])], readMore);
+        }, (err) => {
+            let parsedMsg = ['FULL_HISTORY_END', parsed[1]];
+            if (err) {
+                Log.error('HK_GET_FULL_HISTORY', err.stack);
+                parsedMsg = ['ERROR', parsed[1], err.message];
+            }
+            sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(parsedMsg)]);
+        });
+    };
+
+    const handleRPC = function (ctx, seq, user, parsed) {
+        if (typeof(rpc) !== 'function') { return; }
+
+        /* RPC Calls...  */
+        var rpc_call = parsed.slice(1);
+
+        sendMsg(ctx, user, [seq, 'ACK']);
+        try {
             // slice off the sequence number and pass in the rest of the message
             rpc(ctx, rpc_call, function (err, output) {
                 if (err) {
@@ -992,10 +969,43 @@ module.exports.create = function (cfg) {
                 // finally, send a response to the client that sent the RPC
                 sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0]].concat(output))]);
             });
-            } catch (e) {
-                sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]);
-            }
+        } catch (e) {
+            sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0], 'ERROR', 'SERVER_ERROR'])]);
+        }
+    };
+
+    /*  onDirectMessage
+        * exported for use by the netflux-server
+        * parses and handles all direct messages directed to the history keeper
+          * check if it's expired and execute all the associated side-effects
+          * routes queries to the appropriate handlers
+    */
+    const onDirectMessage = function (ctx, seq, user, json) {
+        Log.silly('HK_MESSAGE', json);
+
+        let parsed;
+        try {
+            parsed = JSON.parse(json[2]);
+        } catch (err) {
+            Log.error("HK_PARSE_CLIENT_MESSAGE", json);
+            return;
+        }
+
+        // If the requested history is for an expired channel, abort
+        // Note the if we don't have the keys for that channel in metadata_cache, we'll
+        // have to abort later (once we know the expiration time)
+        if (checkExpired(ctx, parsed[1])) { return; }
+
+        if (parsed[0] === 'GET_HISTORY') {
+            return void handleGetHistory(ctx, seq, user, parsed);
+        }
+        if (parsed[0] === 'GET_HISTORY_RANGE') {
+            return void handleGetHistoryRange(ctx, seq, user, parsed);
+        }
+        if (parsed[0] === 'GET_FULL_HISTORY') {
+            return void handleGetFullHistory(ctx, seq, user, parsed);
         }
+        return void handleRPC(ctx, seq, user, parsed);
     };
 
     return {

From 66c857d61e7714a7f79f179c3d6429ded826cc41 Mon Sep 17 00:00:00 2001
From: ansuz <ansuz@transitiontech.ca>
Date: Thu, 16 Jan 2020 14:24:07 -0500
Subject: [PATCH 6/6] tiny optimization which saves a little bit of memory
 usage for a little bit of time

---
 historyKeeper.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/historyKeeper.js b/historyKeeper.js
index 1035681d8..fe16a204c 100644
--- a/historyKeeper.js
+++ b/historyKeeper.js
@@ -284,9 +284,9 @@ module.exports.create = function (cfg) {
 
     const storeMessage = function (ctx, channel, msg, isCp, optionalMessageHash) {
         const id = channel.id;
-        const msgBin = Buffer.from(msg + '\n', 'utf8');
 
         queueStorage(id, function (next) {
+            const msgBin = Buffer.from(msg + '\n', 'utf8');
             // Store the message first, and update the index only once it's stored.
             // store.messageBin can be async so updating the index first may
             // result in a wrong cpIndex