Add /api/broadcast and improve message deletion

lib/decrees.js

@@ -24,6 +24,9 @@ SET_PREMIUM_UPLOAD_SIZE
 DISABLE_INTEGRATED_TASKS
 DISABLE_INTEGRATED_EVICTION
 
+// BROADCAST
+SET_LAST_BROADCAST_HASH
+
 NOT IMPLEMENTED:
 
 // RESTRICTED REGISTRATION
@@ -121,6 +124,23 @@ commands.SET_ARCHIVE_RETENTION_TIME = makeIntegerSetter('archiveRetentionTime');
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_ACCOUNT_RETENTION_TIME', [365]]], console.log)
 commands.SET_ACCOUNT_RETENTION_TIME = makeIntegerSetter('accountRetentionTime');
 
+// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_LAST_BROADCAST_HASH', [hash]]], console.log)
+commands.SET_LAST_BROADCAST_HASH = function (Env, args) {
+    if (!Array.isArray(args) || typeof(args[0]) !== "string") {
+        throw new Error('INVALID_ARGS');
+    }
+    if (args[0] && args[0].length !== 64) {
+        throw new Error('INVALID_ARGS');
+    }
+    var hash = args[0];
+    if (hash === Env.lastBroadcastHash) { return false; }
+
+    // Hash is valid and has changed: update it and clear the broadcast cache
+    Env.lastBroadcastHash = hash;
+    Env.broadcastCache = {};
+    return true;
+};
+
 var Quota = require("./commands/quota");
 var Keys = require("./keys");
 var Util = require("./common-util");

lib/env.js

@@ -19,8 +19,10 @@ module.exports.create = function (config) {
         FRESH_MODE: true,
         DEV_MODE: false,
         configCache: {},
+        broadcastCache: {},
         flushCache: function () {
             Env.configCache = {};
+            Env.broadcastCache = {};
             Env.FRESH_KEY = +new Date();
             if (!(Env.DEV_MODE || Env.FRESH_MODE)) { Env.FRESH_MODE = true; }
             if (!Env.Log) { return; }
@@ -65,6 +67,8 @@ module.exports.create = function (config) {
         paths: {},
         //msgStore: config.store,
 
+        lastBroadcastHash: '',
+
         netfluxUsers: {},
 
         pinStore: undefined,

server.js

@@ -109,6 +109,7 @@ var setHeaders = (function () {
 
             // Don't set CSP headers on /api/config because they aren't necessary and they cause problems
             // when duplicated by NGINX in production environments
+            // XXX /api/broadcast too?
             if (/^\/api\/config/.test(req.url)) { return; }
             // targeted CSP, generic policies, maybe custom headers
             const h = [
@@ -268,7 +269,53 @@ var serveConfig = (function () {
     };
 }());
 
+var serveBroadcast = (function () {
+    var cacheString = function () {
+        return (Env.FRESH_KEY? '-' + Env.FRESH_KEY: '') + (Env.DEV_MODE? '-' + (+new Date()): '');
+    };
+
+    var template = function (host) {
+        return [
+            'define(function(){',
+            'return ' + JSON.stringify({
+                lastBroadcastHash: Env.lastBroadcastHash
+            }, null, '\t'),
+            '});'
+        ].join(';\n')
+    };
+
+    var cleanUp = {};
+
+    return function (req, res) {
+        var host = req.headers.host.replace(/\:[0-9]+/, '');
+        res.setHeader('Content-Type', 'text/javascript');
+        // don't cache anything if you're in dev mode
+        if (Env.DEV_MODE) {
+            return void res.send(template(host));
+        }
+        // generate a lookup key for the cache
+        var cacheKey = host + ':' + cacheString();
+
+        // XXX do we need a cache for /api/broadcast?
+        if (!Env.broadcastCache[cacheKey]) {
+            // generate the response and cache it in memory
+            Env.broadcastCache[cacheKey] = template(host);
+            // and create a function to conditionally evict cache entries
+            // which have not been accessed in the last 20 seconds
+            cleanUp[cacheKey] = Util.throttle(function () {
+                delete cleanUp[cacheKey];
+                delete Env.broadcastCache[cacheKey];
+            }, 20000);
+        }
+
+        // successive calls to this function
+        cleanUp[cacheKey]();
+        return void res.send(Env.broadcastCache[cacheKey]);
+    };
+}());
+
 app.get('/api/config', serveConfig);
+app.get('/api/broadcast', serveBroadcast);
 
 var four04_path = Path.resolve(__dirname + '/customize.dist/404.html');
 var custom_four04_path = Path.resolve(__dirname + '/customize/404.html');

www/admin/app-admin.less

@@ -229,6 +229,9 @@
         .cp-broadcast-delete {
             width: 100%;
             min-width: 600px;
+            .empty {
+                font-style: italic;
+            }
             .cp-notification {
                 display: flex;
                 align-items: center;

www/admin/inner.js

@@ -1,6 +1,7 @@
 define([
     'jquery',
     '/api/config',
+    '/customize/application_config.js',
     '/bower_components/chainpad-crypto/crypto.js',
     '/common/toolbar.js',
     '/bower_components/nthen/index.js',
@@ -23,6 +24,7 @@ define([
 ], function (
     $,
     ApiConfig,
+    AppConfig,
     Crypto,
     Toolbar,
     nThen,
@@ -937,7 +939,7 @@ define([
         return;
     };
 
-    // Messages.admin_cat_broadcast // XXX
+    Messages.admin_cat_broadcast = "Broadcast" // XXX
     // Messages.admin_broadcastHint // XXX
     // Messages.admin_broadcastTitle // XXX
     Messages.broadcast_maintenance = 'maintenance';// XXX
@@ -952,9 +954,10 @@ define([
     Messages.broadcast_start = 'Start time';
     Messages.broadcast_end = 'End time';
     Messages.broadcast_preview = "Preview in a fake notification";
-    Messages.broadcast_setLKH = "Mark as latest";
     Messages.broadcast_deleteBtn = "Delete for all";
-    Messages.broadcast_reset = "Reset my visible messages";
+    Messages.broadcast_clear = "Clear all for everybody";
+    Messages.expired = "Expired";
+    Messages.broadcast_empty = "No active message";
 
     var getBroadcastForm = function ($form, key) {
         $form.empty();
@@ -967,16 +970,28 @@ define([
         var button = h('button.btn.btn-primary', Messages.support_formButton);
         var $button = $(button);
 
-        var send = function () {
+        var send = function (_cb) {
+            var cb = Util.once(_cb || function () {});
             var data = getData();
-            if (data === false) { return void UI.warn(Messages.error); }
+            if (data === false) {
+                cb('NODATA');
+                return void UI.warn(Messages.error);
+            }
             $button.prop('disabled', 'disabled');
             data.time = +new Date();
-            common.mailbox.sendTo('BROADCAST_'+key.toUpperCase(), data, {}, function (err) {
+            common.mailbox.sendTo('BROADCAST_'+key.toUpperCase(), data, {}, function (err, data) {
                 $button.prop('disabled', '');
-                if (err) { return UI.warn(Messages.error); }
+                cb(err, data);
+                if (err) {
+                    console.error(err);
+                    return UI.warn(Messages.error);
+                }
+
+                // Clear the UI
                 reset();
-                UI.log(Messages.saved);
+
+                // Only print success if there is no callback
+                if (!_cb) { UI.log(Messages.saved); }
             });
         };
 
@@ -1011,11 +1026,13 @@ define([
 
         if (key === 'custom') {
             (function () {
+                // Custom message
                 var container = h('div.cp-broadcast-container');
                 var $container = $(container);
                 var languages = Messages._languages;
                 var keys = Object.keys(languages).sort();
 
+                // Always keep the textarea ordered by language code
                 var reorder = function () {
                     $container.find('.cp-broadcast-lang').each(function (i, el) {
                         var $el = $(el);
@@ -1023,9 +1040,11 @@ define([
                         $el.css('order', keys.indexOf(l));
                     });
                 };
+                // Remove a textarea
                 var removeLang = function (l) {
                     $container.find('.cp-broadcast-lang[data-lang="'+l+'"]').remove();
                 };
+                // Add a textarea
                 var addLang = function (l) {
                     if ($container.find('.cp-broadcast-lang[data-lang="'+l+'"]').length) { return; }
                     var preview = h('button.btn.btn-secondary', Messages.broadcast_preview);
@@ -1033,6 +1052,8 @@ define([
                         onPreview(l);
                     });
                     var bcastDefault = Messages.broadcast_defaultLanguage;
+                    // XXX 
+                    //var first = !$container.find('.cp-broadcast-lang').length;
                     $container.append(h('div.cp-broadcast-lang', { 'data-lang': l }, [
                         h('h4', languages[l]),
                         h('label', Messages.kanban_body),
@@ -1046,7 +1067,7 @@ define([
                     reorder();
                 };
 
-
+                // Checkboxes to select translations
                 var boxes = keys.map(function (l) {
                     var $cbox = $(UI.createCheckbox('cp-broadcast-custom-lang-'+l,
                         languages[l], false, { label: { class: 'noTitle' } }));
@@ -1063,6 +1084,7 @@ define([
                     return $cbox[0];
                 });
 
+                // Extract form data
                 getData = function () {
                     var map = {};
                     var defaultLanguage;
@@ -1091,11 +1113,13 @@ define([
                         content: map
                     };
                 };
+                // Clear all the textarea when sent
                 reset = function () {
                     $container.find('.cp-broadcast-lang textarea').each(function (i, el) {
                         $(el).val('');
                     });
                 };
+                // Make the form
                 $form.append([
                     h('label', Messages.broadcast_translations),
                     h('div.cp-broadcast-languages', boxes),
@@ -1107,11 +1131,13 @@ define([
         }
         if (key === 'maintenance') {
             (function () {
+                // Maintenance message
+
+                // Start and end date pickers
                 var start = h('input');
                 var end = h('input');
                 var $start = $(start);
                 var $end = $(end);
-
                 var endPickr = Flatpickr(end, {
                     enableTime: true,
                     minDate: new Date()
@@ -1123,6 +1149,8 @@ define([
                         endPickr.set('minDate', new Date($start.val()));
                     }
                 });
+
+                // Extract form data
                 getData = function () {
                     var start = +new Date($start.val());
                     var end = +new Date($start.val());
@@ -1135,6 +1163,8 @@ define([
                         end: end
                     };
                 };
+
+                // Clear when sent
                 reset = function () {
                     $start.val('');
                     $end.val('');
@@ -1153,10 +1183,16 @@ define([
         }
         if (key === 'version') {
             (function () {
+                // New version available message
+
+                // This checkbox can be used to trigger a fake "reconnect" event on the clients
+                // so that they can check api/config and reload the worker in case of a new version
                 var $cbox = $(UI.createCheckbox('cp-admin-version-reload',
                     Messages.broadcast_newVersionReload,
                     false, { label: { class: 'noTitle' } }));
                 var $checkbox = $cbox.find('input');
+
+                // Extract the data and make the form
                 getData = function () {
                     return {
                         reload: $checkbox.is(':checked')
@@ -1176,6 +1212,8 @@ define([
         }
         if (key === 'survey') {
             (function () {
+                // New survey message
+                // TODO send different URLs for other languages?
                 var label = h('label', Messages.broadcast_surveyURL);
                 var input = h('input');
                 var $input = $(input);
@@ -1204,32 +1242,57 @@ define([
         }
 
         if (key === 'delete') {
-            (function () {
+            // Delete form
+            require(['/api/broadcast?'+ (+new Date())], function (BCast) {
+
+            // Always display the messages from the instance "lastBroadcastHash"
+            var hash = BCast.lastBroadcastHash || '1'; // Truthy value if no lastKnownHash
+            common.mailbox.getNotificationsHistory('broadcast', null, hash, function (e, msgs) {
                 var table = h('table.cp-broadcast-delete');
                 var $table = $(table);
-                common.mailbox.subscribe(["broadcast"], {
+
-                    onMessage: function (data, el) {
+                // Empty history
-                        if (Util.find(data, ['content', 'msg', 'type']) === 'BROADCAST_DELETE') {
+                if (!msgs.length) {
+                    $table.append(h('tr', h('td.empty', Messages.broadcast_empty)));
+                }
+
+                // Build the table
+                msgs.forEach(function (data) {
+                    var el = common.mailbox.createElement(data);
+                    var t = Util.find(data, ['content', 'msg', 'type']);
+
+                    // A "DELETE" message is here to disable a previous line
+                    if (t === 'BROADCAST_DELETE') {
                         var _uid = Util.find(data, ['content', 'msg', 'content', 'uid']);
                         var $button = $table.find('[data-uid="'+_uid+'"] td.delete button');
                         $button.prop('disabled', 'disabled').text(Messages.deleted);
                         return;
                     }
+
+                    // Make the line
                     var uid = Util.find(data, ['content', 'msg', 'uid']);
+                    var hash = Util.find(data, ['content', 'hash']);
                     var time = Util.find(data, ['content', 'msg', 'content', 'time']);
-                        var setLKHBtn = h('button.btn.btn-secondary', Messages.broadcast_setLKH);
                     var deleteBtn = h('button.btn.btn-danger', Messages.broadcast_deleteBtn);
-                        $(setLKHBtn).click(function () {
-                            // XXX
-                        });
                     var tr = h('tr', { 'data-uid': uid }, [
                         h('td', 'ID: '+uid),
                         h('td', new Date(time || 0).toLocaleString()),
                         h('td', el),
-                            h('td', setLKHBtn),
                         h('td.delete', deleteBtn),
                     ]);
 
+                    // Auto-expire maintenance and survey messages
+                    if (t === 'BROADCAST_MAINTENANCE') {
+                        var end = Util.find(data, ['content', 'msg', 'content', 'end']);
+                        if (end < +new Date()) {
+                            $(deleteBtn).prop('disabled', 'disabled').text(Messages.expired);
+                        }
+                    }
+                    if (t === 'BROADCAST_VERSION') {
+                        $(deleteBtn).prop('disabled', 'disabled').text(Messages.expired);
+                    }
+
+                    // "Delete this message" button
                     UI.confirmButton(deleteBtn, {
                         classes: 'btn-danger',
                         multiple: true
@@ -1245,23 +1308,44 @@ define([
                     });
 
                     $table.append(tr);
-                    },
-                    history: true // won't receive new messages: not a "subscription"
                 });
 
-                var resetMine = h('button.btn.btn-primary', Messages.broadcast_reset);
+                // Clear all button: remove all the messages and bump lastBroadcastHash
-                UI.confirmButton(resetMine, {}, function () {
+                var clearAll = h('button.btn.btn-danger', Messages.broadcast_clear);
-                    common.mailbox.reset('broadcast', function () {
+                UI.confirmButton(clearAll, {
-                    // XXX
+                    classes: 'btn-danger',
-                        console.error(arguments);
+                    multiple: true
+                }, function () {
+                    getData = function () {
+                        return { all: true };
+                    };
+                    reset = function () {};
+
+                    // Send a message to all users telling them to wipe the broadcast mailbox
+                    // and on success, send an admin decree to update /api/broadcast
+                    send(function (err, obj) {
+                        if (err) { return; }
+                        if (!obj || !obj.hash) { return; }
+                        sFrameChan.query('Q_ADMIN_RPC', {
+                            cmd: 'ADMIN_DECREE',
+                            data: ['SET_LAST_BROADCAST_HASH', [obj.hash]]
+                        }, function (e) {
+                            if (e) {
+                                UI.warn(Messages.error); console.error(e);
+                                return;
+                            }
+                            // On success, reload the "delete" tab
+                            getBroadcastForm($form, key);
+                        });
                     });
                 });
 
                 $form.append([
-                    resetMine,
+                    table,
-                    table
+                    msgs.length ? clearAll : undefined
                 ]);
-            })();
+            });
+            });
             return;
         }
 
@@ -1281,6 +1365,12 @@ define([
             'custom',
             'delete'
         ];
+
+        // The "version" message only works if the instance is using a manual /api/config
+        // This is a custom setup for which our team won't provide support and is NOT
+        // recommended unless you know exactly what you're doing.
+        if (!AppConfig.customApiConfig) { categories.splice(2,1); }
+
         categories = categories.map(function (key) {
             return {
                 tag: 'a',

www/common/notifications.js

@@ -460,7 +460,7 @@ define([
         // Otherwise, fallback to the default language if it exists
         if (!toShow && defaultL) { toShow = text[defaultL]; }
         // No translation available, dismiss
-        if (!toShow) { defaultDismiss(common, data)(); }
+        if (!toShow) { return defaultDismiss(common, data)(); }
 
         var slice = toShow.length > 500;
         toShow = Util.fixHTML(toShow);
@@ -492,6 +492,7 @@ define([
 
     return {
         add: function(common, data) {
+            console.log(data);
             var type = data.content.msg.type;
 
             if (handlers[type]) {

www/common/outer/async-store.js

@@ -2996,7 +2996,8 @@ define([
             /* // XXX NETWORK_RECONNECT only works when a manual /api/config is used
             // XXX The following code disconnect all tabs and asks for a page reload BUT
             // if the urlArgs has not changed, new tabs will stay on the same DISCONNECTED worker
-            // XXX One solution is to change the FRESH mode token before sending the newVersion message
+            // XXX ==> we should probably keep NETWORK_RECONNECT but keep this "version reload" only for us
+            //         because other instances have to reload the server when a new version is deployed
             Store.disconnect();
             broadcast([], "FORCE_RELOAD");
             if (self.CP_closeWorker) {

www/common/outer/mailbox-handlers.js

@@ -742,12 +742,21 @@ define([
     handlers['BROADCAST_DELETE'] = function (ctx, box, data, cb) {
         var msg = data.msg;
         var content = msg.content;
+
+        // If this is a "clear all" message, empty the box and update lkh
+        if (content.all) {
+            // 3rd argument of callback: clear the mailbox
+            return void cb(null, null, true);
+        }
+
         var uid = content.uid; // uid of the message to delete
         if (!broadcasts[uid]) {
             // We don't have this message in memory, nothing to delete
             return void cb(true);
         }
-        cb(false, broadcasts[uid]);
+
+        // We have the message in memory, remove it and don't keep the DELETE msg
+        cb(true, broadcasts[uid]);
         delete broadcasts[uid];
     };
 

www/common/outer/mailbox.js

@@ -1,5 +1,6 @@
 define([
     '/api/config',
+    '/api/broadcast',
     '/common/common-util.js',
     '/common/common-hash.js',
     '/common/common-realtime.js',
@@ -8,7 +9,7 @@ define([
     '/common/outer/mailbox-handlers.js',
     '/bower_components/chainpad-netflux/chainpad-netflux.js',
     '/bower_components/chainpad-crypto/crypto.js',
-], function (Config, Util, Hash, Realtime, Messaging, Notify, Handlers, CpNetflux, Crypto) {
+], function (Config, BCast, Util, Hash, Realtime, Messaging, Notify, Handlers, CpNetflux, Crypto) {
     var Mailbox = {};
 
     var TYPES = [
@@ -46,12 +47,10 @@ define([
         if (!mailboxes['broadcast']) {
             mailboxes.broadcast = {
                 channel: BROADCAST_CHAN,
-                lastKnownHash: '', // XXX load /api/brooadcast to set this hash
+                lastKnownHash: BCast.lastBroadcastHash,
                 decrypted: true,
                 viewed: []
             };
-        } else {
-            // XXX update lastKnownHash from /api/broadcast
         }
     };
 
@@ -159,7 +158,9 @@ proxy.mailboxes = {
                     error: err,
                 });
             }
-            return void cb();
+            return void cb({
+                hash: ciphertext.slice(0,64)
+            });
         });
     };
 
@@ -331,7 +332,22 @@ proxy.mailboxes = {
                     hash: hash
                 };
                 var notify = box.ready;
-                Handlers.add(ctx, box, message, function (dismissed, toDismiss) {
+                Handlers.add(ctx, box, message, function (dismissed, toDismiss, setAsLKH) {
+                    if (setAsLKH) {
+                        // Update LKH
+                        box.data.lastKnownHash = hash;
+                        box.data.viewed = [];
+
+                        // Make sure we remove data about dismissed messages
+                        Realtime.whenRealtimeSyncs(ctx.store.realtime, function () {
+                            Object.keys(box.content).forEach(function (h) {
+                                Handlers.remove(ctx, box, box.content[h], h);
+                                delete box.content[h];
+                                hideMessage(ctx, type, h, ctx.clients);
+                            });
+                        });
+                        return;
+                    }
                     if (toDismiss) { // List of other messages to remove
                         dismiss(ctx, toDismiss, '', function () {
                             console.log('Notification handled automatically');
@@ -422,6 +438,9 @@ proxy.mailboxes = {
             if (type === 'HISTORY_RANGE') {
                 if (!Array.isArray(_msg)) { return; }
                 var message;
+                if (req.box.type === 'broadcast') {
+                    message = Util.tryParse(_msg[4]);
+                } else {
                     try {
                         var decrypted = box.encryptor.decrypt(_msg[4]);
                         message = JSON.parse(decrypted.content);
@@ -429,6 +448,7 @@ proxy.mailboxes = {
                     } catch (e) {
                         console.log(e);
                     }
+                }
                 ctx.emit('HISTORY', {
                     txid: txid,
                     time: _msg[5],
@@ -453,6 +473,13 @@ proxy.mailboxes = {
                 txid: data.txid
             }
         ];
+        if (data.type === 'broadcast') {
+            msg = [ 'GET_HISTORY_RANGE', box.channel, {
+                    to: data.lastKnownHash,
+                    txid: data.txid
+                }
+            ];
+        }
         ctx.req[data.txid] = {
             cId: clientId,
             box: box
@@ -464,21 +491,6 @@ proxy.mailboxes = {
         });
     };
 
-    var resetBox = function (ctx, cId, type, cb) {
-        var box = ctx.mailboxes && ctx.mailboxes[type];
-        if (!box) { return void cb({error: 'ENOENT'}); }
-
-        console.log(box);
-        if (type === 'broadcast') {
-            box.viewed = [];
-            box.lastKnownHash = ''; // XXX Use api/broadcast
-            return void cb();
-        }
-
-        box.lastKnownHash = '';
-        box.viewed = [];
-    };
-
     var subscribe = function (ctx, data, cId, cb) {
         // Get existing notifications
         Object.keys(ctx.boxes).forEach(function (type) {
@@ -521,7 +533,6 @@ proxy.mailboxes = {
             req: {}
         };
 
-
         initializeMailboxes(ctx, mailboxes);
         initializeHistory(ctx);
 
@@ -597,9 +608,6 @@ proxy.mailboxes = {
             if (cmd === 'LOAD_HISTORY') {
                 return void loadHistory(ctx, clientId, data, cb);
             }
-            if (cmd === 'RESET') {
-                return void resetBox(ctx, clientId, data, cb);
-            }
         };
 
         return mailbox;

www/common/sframe-common-mailbox.js

@@ -42,10 +42,10 @@ define([
                 type: type,
                 msg: content,
                 user: user
-            }, function (err, obj) {
+            }, function (obj) {
-                cb(err || (obj && obj.error), obj);
+                cb(obj && obj.error, obj);
-                if (err || (obj && obj.error)) {
+                if (obj && obj.error) {
-                    return void console.error(err || obj.error);
+                    return void console.error(obj.error);
                 }
             });
         };
@@ -226,13 +226,6 @@ define([
             });
         };
 
-        mailbox.reset = function (type, cb) {
-            if (!type) { return; }
-            execCommand('RESET', type, function (obj) {
-                cb(obj);
-            });
-        };
-
         var historyState = false;
         var onHistory = function () {};
         mailbox.getMoreHistory = function (type, count, lastKnownHash, cb) {