Merge branch 'soon' of github.com:xwiki-labs/cryptpad into support-categories

and include some tweaks for the checkup page

* sort errors above warnings
* improve messages for new tests

lib/api.js

@@ -4,6 +4,8 @@ const NetfluxSrv = require('chainpad-server');
 const Decrees = require("./decrees");
 
 const nThen = require("nthen");
+const Fs = require("fs");
+const Path = require("path");
 
 module.exports.create = function (Env) {
     var log = Env.Log;
@@ -19,6 +21,9 @@ nThen(function (w) {
             console.error(err);
         }
     }));
+}).nThen(function (w) {
+    var fullPath = Path.join(Env.paths.block, 'placeholder.txt');
+    Fs.writeFile(fullPath, 'PLACEHOLDER\n', w());
 }).nThen(function () {
     // asynchronously create a historyKeeper and RPC together
     require('./historyKeeper.js').create(Env, function (err, historyKeeper) {

lib/commands/admin-rpc.js

@@ -339,6 +339,9 @@ var instanceStatus = function (Env, Server, cb) {
         disableIntegratedEviction: Env.disableIntegratedEviction,
         disableIntegratedTasks: Env.disableIntegratedTasks,
 
+        enableProfiling: Env.enableProfiling,
+        profilingWindow: Env.profilingWindow,
+
         maxUploadSize: Env.maxUploadSize,
         premiumUploadSize: Env.premiumUploadSize,
 

lib/decrees.js

@@ -24,6 +24,8 @@ SET_PREMIUM_UPLOAD_SIZE
 // BACKGROUND PROCESSES
 DISABLE_INTEGRATED_TASKS
 DISABLE_INTEGRATED_EVICTION
+ENABLE_PROFILING
+SET_PROFILING_WINDOW
 
 // BROADCAST
 SET_LAST_BROADCAST_HASH
@@ -146,6 +148,16 @@ var makeIntegerSetter = function (attr) {
     return makeGenericSetter(attr, args_isInteger);
 };
 
+var arg_isPositiveInteger = function (args) {
+    return Array.isArray(args) && isInteger(args[0]) && args[0] > 0;
+};
+
+// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['ENABLE_PROFILING', [true]]], console.log)
+commands.ENABLE_PROFILING = makeBooleanSetter('enableProfiling');
+
+// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_PROFILING_WINDOW', [10000]]], console.log)
+commands.SET_PROFILING_WINDOW = makeGenericSetter('profilingWindow', arg_isPositiveInteger);
+
 // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_MAX_UPLOAD_SIZE', [50 * 1024 * 1024]]], console.log)
 commands.SET_MAX_UPLOAD_SIZE = makeIntegerSetter('maxUploadSize');
 

lib/env.js

@@ -54,6 +54,10 @@ module.exports.create = function (config) {
 
         launchTime: +new Date(),
 
+        enableProfiling: false,
+        profilingWindow: 10000,
+        bytesWritten: 0,
+
         inactiveTime: config.inactiveTime,
         archiveRetentionTime: config.archiveRetentionTime,
         accountRetentionTime: config.accountRetentionTime,
@@ -226,6 +230,15 @@ module.exports.create = function (config) {
         return typeof(config[key]) === 'string'? config[key]: def;
     };
 
+    Env.incrementBytesWritten = function (n) {
+        if (!Env.enableProfiling) { return; }
+        if (!n || typeof(n) !== 'number' || n < 0) { return; }
+        Env.bytesWritten += n;
+        setTimeout(function () {
+            Env.bytesWritten -= n;
+        }, Env.profilingWindow);
+    };
+
     paths.pin = keyOrDefaultString('pinPath', './pins');
     paths.block = keyOrDefaultString('blockPath', './block');
     paths.data = keyOrDefaultString('filePath', './datastore');

lib/hk-util.js

@@ -380,10 +380,14 @@ const storeMessage = function (Env, channel, msg, isCp, optionalMessageHash, cb)
                 // Message stored, call back
                 cb();
 
-                index.size += msgBin.length;
+                var msgLength = msgBin.length;
+                index.size += msgLength;
 
                 // handle the next element in the queue
                 next();
+
+                // keep track of how many bytes are written
+                Env.incrementBytesWritten(msgLength);
             }));
         });
     });

lib/storage/blob.js

@@ -131,11 +131,13 @@ var upload = function (Env, safeKey, content, cb) {
             blobstage.write(dec);
             session.currentUploadSize += len;
             cb(void 0, dec.length);
+            //Env.incrementBytesWritten(len);
         });
     } else {
         session.blobstage.write(dec);
         session.currentUploadSize += len;
         cb(void 0, dec.length);
+        //Env.incrementBytesWritten(len);
     }
 };
 
@@ -493,6 +495,10 @@ BlobStore.create = function (config, _cb) {
         Fse.mkdirp(Path.join(Env.archivePath, Env.blobPath), w(function (e) {
             if (e) { CB(e); }
         }));
+    }).nThen(function (w) {
+        // XXX make a placeholder file in the root of the blob path
+        var fullPath = Path.join(Env.blobPath, 'placeholder.txt');
+        Fse.writeFile(fullPath, 'PLACEHOLDER\n', w());
     }).nThen(function () {
         var methods = {
             isFileId: isValidId,

lib/storage/block.js

@@ -49,6 +49,7 @@ Block.archive = function (Env, publicKey, _cb) {
         return void cb('E_INVALID_BLOCK_ARCHIVAL_PATH');
     }
 
+    // TODO Env.incrementBytesWritten
     Fse.move(currentPath, archivePath, {
         overwrite: true,
     }, cb);
@@ -83,6 +84,7 @@ Block.write = function (Env, publicKey, buffer, _cb) {
         }));
     }).nThen(function () {
         Fs.writeFile(path, buffer, { encoding: 'binary' }, cb);
+        Env.incrementBytesWritten(buffer && buffer.length);
     });
 };
 

server.js

@@ -284,6 +284,13 @@ var send404 = function (res, path) {
         send404(res);
     });
 };
+app.get('/api/profiling', function (req, res, next) {
+    if (!Env.enableProfiling) { return void send404(res); }
+    res.setHeader('Content-Type', 'text/javascript');
+    res.send(JSON.stringify({
+        bytesWritten: Env.bytesWritten,
+    }));
+});
 
 app.use(function (req, res, next) {
     res.status(404);

www/admin/inner.js

@@ -70,6 +70,7 @@ define([
         ],
         'stats': [ // Msg.admin_cat_stats
             'cp-admin-refresh-stats',
+            'cp-admin-uptime',
             'cp-admin-active-sessions',
             'cp-admin-active-pads',
             'cp-admin-open-files',
@@ -89,6 +90,8 @@ define([
         'performance': [ // Msg.admin_cat_performance
             'cp-admin-refresh-performance',
             'cp-admin-performance-profiling',
+            'cp-admin-enable-disk-measurements',
+            'cp-admin-bytes-written',
         ],
         'network': [ // Msg.admin_cat_network
             'cp-admin-update-available',
@@ -793,6 +796,29 @@ define([
         return $div;
     };
 
+    Messages.admin_uptimeTitle = 'Launch time';
+    Messages.admin_uptimeHint = 'Date and time at which the server was launched';
+
+    create['uptime'] = function () {
+        var key = 'uptime';
+        var $div = makeBlock(key); // Msg.admin_activeSessionsHint, .admin_activeSessionsTitle
+        var pre = h('pre');
+
+        var set = function () {
+            var uptime = APP.instanceStatus.launchTime;
+            if (typeof(uptime) !== 'number') { return; }
+            pre.innerText = new Date(uptime);
+        };
+
+        set();
+
+        $div.append(pre);
+        onRefreshStats.reg(function () {
+            set();
+        });
+        return $div;
+    };
+
     create['active-sessions'] = function () {
         var key = 'active-sessions';
         var $div = makeBlock(key); // Msg.admin_activeSessionsHint, .admin_activeSessionsTitle
@@ -1888,6 +1914,84 @@ define([
         return $div;
     };
 
+    Messages.admin_enableDiskMeasurementsTitle = "Measure disk performance"; // XXX
+    Messages.admin_enableDiskMeasurementsHint = "If enabled, a JSON endpoint will be exposed under /api/profiling which keeps a running measurement of disk I/O within a configurable window. This setting can impact server performance and may reveal data you'd rather keep hidden. It is recommended that you leave it disabled unless you know what you are doing."; // XXX
+
+    create['enable-disk-measurements'] = makeAdminCheckbox({
+        key: 'enable-disk-measurements',
+        getState: function () {
+            return APP.instanceStatus.enableProfiling;
+        },
+        query: function (val, setState) {
+            sFrameChan.query('Q_ADMIN_RPC', {
+                cmd: 'ADMIN_DECREE',
+                data: ['ENABLE_PROFILING', [val]]
+            }, function (e, response) {
+                if (e || response.error) {
+                    UI.warn(Messages.error);
+                    console.error(e, response);
+                }
+                APP.updateStatus(function () {
+                    setState(APP.instanceStatus.enableProfiling);
+                });
+            });
+        },
+    });
+
+    Messages.admin_bytesWrittenTitle = "Disk performance measurement window";
+    Messages.admin_bytesWrittenHint = "If you have enabled disk performance measurements then the duration of the window can be configured below."; // XXX
+    Messages.admin_bytesWrittenDuration = "Duration of the window in milliseconds: {0}"; // XXX
+    Messages.admin_defaultDuration = "admin_defaultDuration"; // XXX
+    Messages.admin_setDuration = "Set duration"; // XXX
+
+    var isPositiveInteger = function (n) {
+        return n && typeof(n) === 'number'  && n % 1 === 0 && n > 0;
+    };
+
+    create['bytes-written'] = function () {
+        var key = 'bytes-written';
+        var $div = makeBlock(key);
+
+        var duration = APP.instanceStatus.profilingWindow;
+        if (!isPositiveInteger(duration)) { duration = 10000; }
+        var newDuration = h('input', {type: 'number', min: 0, value: duration});
+        var set = h('button.btn.btn-primary', Messages.admin_setDuration);
+        $div.append(h('div', [
+            h('span.cp-admin-bytes-written-duration', Messages._getKey('admin_bytesWrittenDuration', [duration])),
+            h('div.cp-admin-setlimit-form', [
+                newDuration,
+                h('nav', [set])
+            ])
+        ]));
+
+        UI.confirmButton(set, {
+            classes: 'btn-primary',
+            multiple: true,
+            validate: function () {
+                var l = parseInt($(newDuration).val());
+                if (isNaN(l)) { return false; }
+                return true;
+            }
+        }, function () {
+            var d = parseInt($(newDuration).val());
+            if (!isPositiveInteger(d)) { return void UI.warn(Messages.error); }
+
+            var data = [d];
+            sFrameChan.query('Q_ADMIN_RPC', {
+                cmd: 'ADMIN_DECREE',
+                data: ['SET_PROFILING_WINDOW', data]
+            }, function (e, response) {
+                if (e || response.error) {
+                    UI.warn(Messages.error);
+                    return void console.error(e, response);
+                }
+                $div.find('.cp-admin-bytes-written-duration').text(Messages._getKey('admin_limit', [d]));
+            });
+        });
+
+        return $div;
+    };
+
     create['update-available'] = function () { // Messages.admin_updateAvailableTitle.admin_updateAvailableHint.admin_updateAvailableLabel.admin_updateAvailableButton
         if (!APP.instanceStatus.updateAvailable) { return; }
         var $div = makeBlock('update-available', true);

www/checkup/main.js

@@ -1113,9 +1113,22 @@ define([
         });
     });
 
-    var POLICY_ADVISORY = " It's advised that you either provide one or disable registration.";
+    var POLICY_ADVISORY = " This link will be included in the home page footer and 'About CryptPad' menu. It's advised that you either provide one or disable registration.";
+    var APPCONFIG_DOCS_LINK = function (key) {
+        return h('span', [
+            " See ",
+            h('a', {
+                href: 'https://docs.cryptpad.fr/en/admin_guide/customization.html#application-config',
+                target: "_blank",
+                rel: 'noopener noreferrer',
+            }, "the relevant documentation"),
+            " about how to customize CryptPad's ",
+            code(key),
+            ' value.',
+        ]);
+    };
+
     var isValidInfoURL = function (url) {
-        // XXX check that it's an absolute URL ????
         if (!url || typeof(url) !== 'string') { return false; }
         try {
             var parsed = new URL(url, ApiConfig.httpUnsafeOrigin);
@@ -1130,56 +1143,108 @@ define([
         }
     };
 
-    // XXX check if they provide terms of service
+    // check if they provide terms of service
     assert(function (cb, msg) {
         if (ApiConfig.restrictRegistration) { return void cb(true); }
 
         var url = Pages.customURLs.terms;
         setWarningClass(msg);
         msg.appendChild(h('span', [
-            'No terms of service specified.', // XXX
+            'No terms of service were specified.',
             POLICY_ADVISORY,
+            APPCONFIG_DOCS_LINK('terms'),
         ]));
-        cb(isValidInfoURL(url) || url); // XXX
+        cb(isValidInfoURL(url) || url);
     });
 
-    // XXX check if they provide legal data
+    // check if they provide legal data
     assert(function (cb, msg) {
         if (ApiConfig.restrictRegistration) { return void cb(true); }
 
         var url = Pages.customURLs.imprint;
         setWarningClass(msg);
         msg.appendChild(h('span', [
-            'No legal data provided.', // XXX
+            'No legal entity data was specified.',
             POLICY_ADVISORY,
+            APPCONFIG_DOCS_LINK('imprint'),
         ]));
-        cb(isValidInfoURL(url) || url); // XXX
+        cb(isValidInfoURL(url) || url);
     });
 
-    // XXX check if they provide a privacy policy
+    // check if they provide a privacy policy
     assert(function (cb, msg) {
         if (ApiConfig.restrictRegistration) { return void cb(true); }
 
         var url = Pages.customURLs.privacy;
         setWarningClass(msg);
         msg.appendChild(h('span', [
-            'No privacy policy provided.', // XXX
+            'No privacy policy was specified.',
             POLICY_ADVISORY,
+            APPCONFIG_DOCS_LINK('privacy'),
         ]));
-        cb(isValidInfoURL(url) || url); // XXX
+        cb(isValidInfoURL(url) || url);
     });
 
-    // XXX check if they provide a link to source code
+    // check if they provide a link to source code
     assert(function (cb, msg) {
         if (ApiConfig.restrictRegistration) { return void cb(true); }
 
         var url = Pages.customURLs.source;
         setWarningClass(msg);
         msg.appendChild(h('span', [
-            'No source code link provided.', // XXX
+            'No source code link was specified.',
             POLICY_ADVISORY,
+            APPCONFIG_DOCS_LINK('source'),
+        ]));
+        cb(isValidInfoURL(url) || url);
+    });
+
+    assert(function (cb, msg) {
+        var path = '/blob/placeholder.txt';
+        var fullPath;
+        try {
+            fullPath = new URL(path, ApiConfig.fileHost || ApiConfig.httpUnsafeOrigin).href;
+        } catch (err) {
+            fullPath = path;
+        }
+
+        msg.appendChild(h('span', [
+            "A placeholder file was expected to be available at ",
+            code(fullPath),
+            ", but it was not found.",
+            " This commonly indicates a mismatch between the API server's ",
+            code('blobPath'),
+            " value and the path that the webserver or reverse proxy is attempting to serve.",
+            " This misconfiguration will cause errors with uploaded files and CryptPad's office editors (sheet, presentation, document).",
+        ]));
+
+        Tools.common_xhr(fullPath, xhr => {
+            cb(xhr.status === 200 || xhr.status);
+        });
+    });
+
+    assert(function (cb, msg) {
+        var path = '/block/placeholder.txt';
+        var fullPath;
+        try {
+            fullPath = new URL(path, ApiConfig.fileHost || ApiConfig.httpUnsafeOrigin).href;
+        } catch (err) {
+            fullPath = path;
+        }
+
+        msg.appendChild(h('span', [
+            "A placeholder file was expected to be available at ",
+            code(fullPath),
+            ", but it was not found.",
+            " This commonly indicates a mismatch between the API server's ",
+            code('blockPath'),
+            " value and the path that the webserver or reverse proxy is attempting to serve.",
+            " This misconfiguration will cause errors with login, registration, and password change.",
         ]));
-        cb(isValidInfoURL(url) || url); // XXX
+
+        Tools.common_xhr(fullPath, xhr => {
+            cb(xhr.status === 200 || xhr.status);
+        });
     });
 
     var serverToken;
@@ -1271,9 +1336,20 @@ define([
             details,
         ]);
 
+        var isWarning = function (x) {
+            return x && /cp\-warning/.test(x.getAttribute('class'));
+        };
+
+        var sortMethod = function (a, b) {
+            if (isWarning(a.message) && !isWarning(b.message)) {
+                return 1;
+            }
+            return a.test - b.test;
+        };
+
         var report = h('div.report', [
             summary,
-            h('div.failures', errors.map(failureReport)),
+            h('div.failures', errors.sort(sortMethod).map(failureReport)),
         ]);
 
         $progress.remove();

www/common/outer/async-store.js

@@ -1161,6 +1161,7 @@ define([
 
             //var storeLocally = data.teamId === -1;
             if (data.teamId === -1) { data.teamId = undefined; }
+            if (data.teamId) { data.teamId = Number(data.teamId); }
 
             // If a teamId is provided, it means we want to store the pad in a specific
             // team drive. In this case, we just need to check if the pad is already

www/common/translations/messages.fr.json

@@ -1436,5 +1436,17 @@
     "admin_descriptionTitle": "Description de l'instance",
     "admin_nameHint": "Le nom affiché pour cette instance dans la liste des instances publiques sur cryptpad.org",
     "admin_nameTitle": "Nom de l'instance",
-    "admin_archiveNote": "Note"
+    "admin_archiveNote": "Note",
+    "support_cat_abuse": "Signaler un abus",
+    "support_cat_document": "Document",
+    "support_cat_drives": "Drive ou équipe",
+    "support_warning_other": "Quelle est la nature de votre demande ? Veuillez fournir autant d'informations pertinentes que possible afin de nous permettre de traiter rapidement votre problème",
+    "support_warning_abuse": "Merci de signaler les contenus qui ne respectent pas les <a>Conditions d'utilisation</a>. Veuillez fournir des liens vers les documents ou les profils d'utilisateurs incriminés et décrire en quoi ils enfreignent les conditions. Toute information supplémentaire sur le contexte dans lequel vous avez découvert le contenu ou le comportement peut aider les administrateurs à prévenir de futures violations",
+    "support_warning_bug": "Veuillez préciser dans quel navigateur le problème se produit et si des extensions sont installées. Veuillez fournir autant de détails que possible sur le problème et les étapes nécessaires pour le reproduire",
+    "support_warning_document": "Veuillez préciser quel type de document est à l'origine du problème et fournir la <a>référence du document</a> ou un lien",
+    "support_warning_drives": "Veuillez noter que les administrateurs ne sont pas en mesure d'identifier des dossiers ou documents à partir de leur nom. Pour les dossiers partagés, veuillez fournir la <a>référence du dossier</a>",
+    "support_warning_account": "Veuillez noter que les administrateurs ne sont pas en mesure de réinitialiser les mots de passe. Si vous avez perdu vos identifiants mais que vous êtes encore connecté, vous pouvez <a>transférer vos données vers un nouveau compte</a>",
+    "support_warning_prompt": "Merci de choisir la catégorie la plus pertinente pour qualifier votre problème, ceci aide les administrateurs à faire le tri et fournit des suggestions sur les informations à fournir",
+    "admin_jurisdictionTitle": "Pays d'hébergement",
+    "ui_saved": "{0} enregistré"
 }

www/profile/inner.js

@@ -135,6 +135,14 @@ define([
             rel: 'noreferrer noopener'
         }).appendTo($block).hide();
 
+        APP.$link.click(function (ev) {
+            ev.preventDefault();
+            ev.stopPropagation();
+            var href = $(this).attr('href').trim();
+            if (!href) { return; }
+            common.openUnsafeURL(href);
+        });
+
         APP.$linkEdit = $();
         if (APP.readOnly) { return; }