From 6a2ec4ae7e24814125e5a6c429713fa09d009287 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 25 Feb 2022 13:14:59 +0530 Subject: [PATCH 1/7] consider profile links unsafe --- www/profile/inner.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/www/profile/inner.js b/www/profile/inner.js index 8f7b8cf08..230bcc88e 100644 --- a/www/profile/inner.js +++ b/www/profile/inner.js @@ -135,6 +135,14 @@ define([ rel: 'noreferrer noopener' }).appendTo($block).hide(); + APP.$link.click(function (ev) { + ev.preventDefault(); + ev.stopPropagation(); + var href = $(this).attr('href').trim(); + if (!href) { return; } + common.openUnsafeURL(href); + }); + APP.$linkEdit = $(); if (APP.readOnly) { return; } From b65730b853036b181041f010cd52980624c6ba88 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 7 Mar 2022 18:42:00 +0530 Subject: [PATCH 2/7] allow admins to enable configurable disk I/O profiling --- lib/commands/admin-rpc.js | 3 ++ lib/decrees.js | 12 +++++ lib/env.js | 13 +++++ lib/hk-util.js | 6 ++- lib/storage/blob.js | 2 + lib/storage/block.js | 2 + server.js | 7 +++ www/admin/inner.js | 104 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 148 insertions(+), 1 deletion(-) diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 59750571e..3fc87bce5 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -315,6 +315,9 @@ var instanceStatus = function (Env, Server, cb) { disableIntegratedEviction: Env.disableIntegratedEviction, disableIntegratedTasks: Env.disableIntegratedTasks, + enableProfiling: Env.enableProfiling, + profilingWindow: Env.profilingWindow, + maxUploadSize: Env.maxUploadSize, premiumUploadSize: Env.premiumUploadSize, diff --git a/lib/decrees.js b/lib/decrees.js index 5f599705e..371a4f8d7 100644 --- a/lib/decrees.js +++ b/lib/decrees.js @@ -24,6 +24,8 @@ SET_PREMIUM_UPLOAD_SIZE // BACKGROUND PROCESSES DISABLE_INTEGRATED_TASKS DISABLE_INTEGRATED_EVICTION +ENABLE_PROFILING +SET_PROFILING_WINDOW // BROADCAST SET_LAST_BROADCAST_HASH @@ -143,6 +145,16 @@ var makeIntegerSetter = function (attr) { return makeGenericSetter(attr, args_isInteger); }; +var arg_isPositiveInteger = function (args) { + return Array.isArray(args) && isInteger(args[0]) && args[0] > 0; +}; + +// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['ENABLE_PROFILING', [true]]], console.log) +commands.ENABLE_PROFILING = makeBooleanSetter('enableProfiling'); + +// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_PROFILING_WINDOW', [10000]]], console.log) +commands.SET_PROFILING_WINDOW = makeGenericSetter('profilingWindow', arg_isPositiveInteger); + // CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_MAX_UPLOAD_SIZE', [50 * 1024 * 1024]]], console.log) commands.SET_MAX_UPLOAD_SIZE = makeIntegerSetter('maxUploadSize'); diff --git a/lib/env.js b/lib/env.js index 9970bc4f9..656855396 100644 --- a/lib/env.js +++ b/lib/env.js @@ -54,6 +54,10 @@ module.exports.create = function (config) { launchTime: +new Date(), + enableProfiling: false, + profilingWindow: 10000, + bytesWritten: 0, + inactiveTime: config.inactiveTime, archiveRetentionTime: config.archiveRetentionTime, accountRetentionTime: config.accountRetentionTime, @@ -222,6 +226,15 @@ module.exports.create = function (config) { return typeof(config[key]) === 'string'? config[key]: def; }; + Env.incrementBytesWritten = function (n) { + if (!Env.enableProfiling) { return; } + if (!n || typeof(n) !== 'number' || n < 0) { return; } + Env.bytesWritten += n; + setTimeout(function () { + Env.bytesWritten -= n; + }, Env.profilingWindow); + }; + paths.pin = keyOrDefaultString('pinPath', './pins'); paths.block = keyOrDefaultString('blockPath', './block'); paths.data = keyOrDefaultString('filePath', './datastore'); diff --git a/lib/hk-util.js b/lib/hk-util.js index 276ac3e6f..a95eb19a6 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -380,10 +380,14 @@ const storeMessage = function (Env, channel, msg, isCp, optionalMessageHash, cb) // Message stored, call back cb(); - index.size += msgBin.length; + var msgLength = msgBin.length; + index.size += msgLength; // handle the next element in the queue next(); + + // keep track of how many bytes are written + Env.incrementBytesWritten(msgLength); })); }); }); diff --git a/lib/storage/blob.js b/lib/storage/blob.js index e5c7a2fce..a48f669af 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -131,11 +131,13 @@ var upload = function (Env, safeKey, content, cb) { blobstage.write(dec); session.currentUploadSize += len; cb(void 0, dec.length); + Env.incrementBytesWritten(len); }); } else { session.blobstage.write(dec); session.currentUploadSize += len; cb(void 0, dec.length); + Env.incrementBytesWritten(len); } }; diff --git a/lib/storage/block.js b/lib/storage/block.js index 1078f6d2e..90228c6bc 100644 --- a/lib/storage/block.js +++ b/lib/storage/block.js @@ -49,6 +49,7 @@ Block.archive = function (Env, publicKey, _cb) { return void cb('E_INVALID_BLOCK_ARCHIVAL_PATH'); } + // TODO Env.incrementBytesWritten Fse.move(currentPath, archivePath, { overwrite: true, }, cb); @@ -83,6 +84,7 @@ Block.write = function (Env, publicKey, buffer, _cb) { })); }).nThen(function () { Fs.writeFile(path, buffer, { encoding: 'binary' }, cb); + Env.incrementBytesWritten(buffer && buffer.length); }); }; diff --git a/server.js b/server.js index 73f63a3e1..204cd5f6e 100644 --- a/server.js +++ b/server.js @@ -284,6 +284,13 @@ var send404 = function (res, path) { send404(res); }); }; +app.get('/api/profiling', function (req, res, next) { + if (!Env.enableProfiling) { return void send404(res); } + res.setHeader('Content-Type', 'text/javascript'); + res.send(JSON.stringify({ + bytesWritten: Env.bytesWritten, + })); +}); app.use(function (req, res, next) { res.status(404); diff --git a/www/admin/inner.js b/www/admin/inner.js index 98213b384..59e410b55 100644 --- a/www/admin/inner.js +++ b/www/admin/inner.js @@ -66,6 +66,7 @@ define([ ], 'stats': [ // Msg.admin_cat_stats 'cp-admin-refresh-stats', + 'cp-admin-uptime', 'cp-admin-active-sessions', 'cp-admin-active-pads', 'cp-admin-open-files', @@ -85,6 +86,8 @@ define([ 'performance': [ // Msg.admin_cat_performance 'cp-admin-refresh-performance', 'cp-admin-performance-profiling', + 'cp-admin-enable-disk-measurements', + 'cp-admin-bytes-written', ], 'network': [ // Msg.admin_cat_network 'cp-admin-update-available', @@ -644,6 +647,29 @@ define([ return $div; }; + Messages.admin_uptimeTitle = 'Launch time'; + Messages.admin_uptimeHint = 'Date and time at which the server was launched'; + + create['uptime'] = function () { + var key = 'uptime'; + var $div = makeBlock(key); // Msg.admin_activeSessionsHint, .admin_activeSessionsTitle + var pre = h('pre'); + + var set = function () { + var uptime = APP.instanceStatus.launchTime; + if (typeof(uptime) !== 'number') { return; } + pre.innerText = new Date(uptime); + }; + + set(); + + $div.append(pre); + onRefreshStats.reg(function () { + set(); + }); + return $div; + }; + create['active-sessions'] = function () { var key = 'active-sessions'; var $div = makeBlock(key); // Msg.admin_activeSessionsHint, .admin_activeSessionsTitle @@ -1739,6 +1765,84 @@ define([ return $div; }; + Messages.admin_enableDiskMeasurementsTitle = "Measure disk performance"; // XXX + Messages.admin_enableDiskMeasurementsHint = "If enabled, a JSON endpoint will be exposed under /api/profiling which keeps a running measurement of disk I/O within a configurable window. This setting can impact server performance and may reveal data you'd rather keep hidden. It is recommended that you leave it disabled unless you know what you are doing."; // XXX + + create['enable-disk-measurements'] = makeAdminCheckbox({ + key: 'enable-disk-measurements', + getState: function () { + return APP.instanceStatus.enableProfiling; + }, + query: function (val, setState) { + sFrameChan.query('Q_ADMIN_RPC', { + cmd: 'ADMIN_DECREE', + data: ['ENABLE_PROFILING', [val]] + }, function (e, response) { + if (e || response.error) { + UI.warn(Messages.error); + console.error(e, response); + } + APP.updateStatus(function () { + setState(APP.instanceStatus.enableProfiling); + }); + }); + }, + }); + + Messages.admin_bytesWrittenTitle = "Disk performance measurement window"; + Messages.admin_bytesWrittenHint = "If you have enabled disk performance measurements then the duration of the window can be configured below."; // XXX + Messages.admin_bytesWrittenDuration = "Duration of the window in milliseconds: {0}"; // XXX + Messages.admin_defaultDuration = "admin_defaultDuration"; // XXX + Messages.admin_setDuration = "Set duration"; // XXX + + var isPositiveInteger = function (n) { + return n && typeof(n) === 'number' && n % 1 === 0 && n > 0; + }; + + create['bytes-written'] = function () { + var key = 'bytes-written'; + var $div = makeBlock(key); + + var duration = APP.instanceStatus.profilingWindow; + if (!isPositiveInteger(duration)) { duration = 10000; } + var newDuration = h('input', {type: 'number', min: 0, value: duration}); + var set = h('button.btn.btn-primary', Messages.admin_setDuration); + $div.append(h('div', [ + h('span.cp-admin-bytes-written-duration', Messages._getKey('admin_bytesWrittenDuration', [duration])), + h('div.cp-admin-setlimit-form', [ + newDuration, + h('nav', [set]) + ]) + ])); + + UI.confirmButton(set, { + classes: 'btn-primary', + multiple: true, + validate: function () { + var l = parseInt($(newDuration).val()); + if (isNaN(l)) { return false; } + return true; + } + }, function () { + var d = parseInt($(newDuration).val()); + if (!isPositiveInteger(d)) { return void UI.warn(Messages.error); } + + var data = [d]; + sFrameChan.query('Q_ADMIN_RPC', { + cmd: 'ADMIN_DECREE', + data: ['SET_PROFILING_WINDOW', data] + }, function (e, response) { + if (e || response.error) { + UI.warn(Messages.error); + return void console.error(e, response); + } + $div.find('.cp-admin-bytes-written-duration').text(Messages._getKey('admin_limit', [d])); + }); + }); + + return $div; + }; + create['update-available'] = function () { // Messages.admin_updateAvailableTitle.admin_updateAvailableHint.admin_updateAvailableLabel.admin_updateAvailableButton if (!APP.instanceStatus.updateAvailable) { return; } var $div = makeBlock('update-available', true); From c86ffead3cd7116694a77e06b7ab7df22e4672a3 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 7 Mar 2022 15:24:44 +0100 Subject: [PATCH 3/7] Fix duplicate pads in teams --- www/common/outer/async-store.js | 1 + 1 file changed, 1 insertion(+) diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index 0856572bc..5d8eac183 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -1161,6 +1161,7 @@ define([ //var storeLocally = data.teamId === -1; if (data.teamId === -1) { data.teamId = undefined; } + if (data.teamId) { data.teamId = Number(data.teamId); } // If a teamId is provided, it means we want to store the pad in a specific // team drive. In this case, we just need to check if the pad is already From 46e9c6bc05c5263b359294486b860dc4885e3304 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 8 Mar 2022 10:41:15 +0530 Subject: [PATCH 4/7] lint compliance --- www/admin/inner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/admin/inner.js b/www/admin/inner.js index 84ff8f5ea..99d640940 100644 --- a/www/admin/inner.js +++ b/www/admin/inner.js @@ -267,7 +267,7 @@ define([ UI.log(archive ? Messages.archivedFromServer : Messages.restoredFromServer); $input.val(''); $pwInput.val(''); - $reason.val('') + $reason.val(''); }); }); }); From ef398de4a1cdd90e2ffcf870962ef9ff9f9422c8 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 8 Mar 2022 11:03:32 +0530 Subject: [PATCH 5/7] disable measurement --- lib/storage/blob.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/storage/blob.js b/lib/storage/blob.js index a48f669af..4b17dc85c 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -131,13 +131,13 @@ var upload = function (Env, safeKey, content, cb) { blobstage.write(dec); session.currentUploadSize += len; cb(void 0, dec.length); - Env.incrementBytesWritten(len); + //Env.incrementBytesWritten(len); }); } else { session.blobstage.write(dec); session.currentUploadSize += len; cb(void 0, dec.length); - Env.incrementBytesWritten(len); + //Env.incrementBytesWritten(len); } }; From c111364024d33faf408f2111e10f9e6fed82b92a Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 8 Mar 2022 18:20:11 +0530 Subject: [PATCH 6/7] add two new tests to diagnose unavailability of uploaded blocks and blobs --- lib/api.js | 5 +++++ lib/storage/blob.js | 4 ++++ www/checkup/main.js | 48 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/lib/api.js b/lib/api.js index 277d085fa..9a317be86 100644 --- a/lib/api.js +++ b/lib/api.js @@ -4,6 +4,8 @@ const NetfluxSrv = require('chainpad-server'); const Decrees = require("./decrees"); const nThen = require("nthen"); +const Fs = require("fs"); +const Path = require("path"); module.exports.create = function (Env) { var log = Env.Log; @@ -19,6 +21,9 @@ nThen(function (w) { console.error(err); } })); +}).nThen(function (w) { + var fullPath = Path.join(Env.paths.block, 'placeholder.txt'); + Fs.writeFile(fullPath, 'PLACEHOLDER\n', w()); }).nThen(function () { // asynchronously create a historyKeeper and RPC together require('./historyKeeper.js').create(Env, function (err, historyKeeper) { diff --git a/lib/storage/blob.js b/lib/storage/blob.js index 4b17dc85c..7cf13dd59 100644 --- a/lib/storage/blob.js +++ b/lib/storage/blob.js @@ -495,6 +495,10 @@ BlobStore.create = function (config, _cb) { Fse.mkdirp(Path.join(Env.archivePath, Env.blobPath), w(function (e) { if (e) { CB(e); } })); + }).nThen(function (w) { + // XXX make a placeholder file in the root of the blob path + var fullPath = Path.join(Env.blobPath, 'placeholder.txt'); + Fse.writeFile(fullPath, 'PLACEHOLDER\n', w()); }).nThen(function () { var methods = { isFileId: isValidId, diff --git a/www/checkup/main.js b/www/checkup/main.js index 2c227df5d..e58976a8a 100644 --- a/www/checkup/main.js +++ b/www/checkup/main.js @@ -1113,6 +1113,54 @@ define([ }); }); + assert(function (cb, msg) { + var path = '/blob/placeholder.txt'; + var fullPath; + try { + fullPath = new URL(path, ApiConfig.fileHost || ApiConfig.httpUnsafeOrigin).href; + } catch (err) { + fullPath = path; + } + + msg.appendChild(h('span', [ + "A placeholder file was expected to be available at ", + code(fullPath), + ", but it was not found.", + " This commonly indicates a mismatch between the API server's ", + code('blobPath'), + " value and the path that the webserver or reverse proxy is attempting to serve.", + " This misconfiguration will cause errors with uploaded files and CryptPad's office editors (sheet, presentation, document).", + ])); + + Tools.common_xhr(fullPath, xhr => { + cb(xhr.status === 200 || xhr.status); + }); + }); + + assert(function (cb, msg) { + var path = '/block/placeholder.txt'; + var fullPath; + try { + fullPath = new URL(path, ApiConfig.fileHost || ApiConfig.httpUnsafeOrigin).href; + } catch (err) { + fullPath = path; + } + + msg.appendChild(h('span', [ + "A placeholder file was expected to be available at ", + code(fullPath), + ", but it was not found.", + " This commonly indicates a mismatch between the API server's ", + code('blockPath'), + " value and the path that the webserver or reverse proxy is attempting to serve.", + " This misconfiguration will cause errors with login, registration, and password change.", + ])); + + Tools.common_xhr(fullPath, xhr => { + cb(xhr.status === 200 || xhr.status); + }); + }); + var serverToken; Tools.common_xhr('/', function (xhr) { serverToken = xhr.getResponseHeader('server'); From 5de0e4891805005cd2f39518f40a1a73d8252a11 Mon Sep 17 00:00:00 2001 From: Weblate Date: Tue, 8 Mar 2022 16:04:56 +0100 Subject: [PATCH 7/7] Translated using Weblate (French) Currently translated at 100.0% (1448 of 1448 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/fr/ --- www/common/translations/messages.fr.json | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/www/common/translations/messages.fr.json b/www/common/translations/messages.fr.json index 5a7f8e22b..59746600b 100644 --- a/www/common/translations/messages.fr.json +++ b/www/common/translations/messages.fr.json @@ -1436,5 +1436,17 @@ "admin_descriptionTitle": "Description de l'instance", "admin_nameHint": "Le nom affiché pour cette instance dans la liste des instances publiques sur cryptpad.org", "admin_nameTitle": "Nom de l'instance", - "admin_archiveNote": "Note" + "admin_archiveNote": "Note", + "support_cat_abuse": "Signaler un abus", + "support_cat_document": "Document", + "support_cat_drives": "Drive ou équipe", + "support_warning_other": "Quelle est la nature de votre demande ? Veuillez fournir autant d'informations pertinentes que possible afin de nous permettre de traiter rapidement votre problème", + "support_warning_abuse": "Merci de signaler les contenus qui ne respectent pas les Conditions d'utilisation. Veuillez fournir des liens vers les documents ou les profils d'utilisateurs incriminés et décrire en quoi ils enfreignent les conditions. Toute information supplémentaire sur le contexte dans lequel vous avez découvert le contenu ou le comportement peut aider les administrateurs à prévenir de futures violations", + "support_warning_bug": "Veuillez préciser dans quel navigateur le problème se produit et si des extensions sont installées. Veuillez fournir autant de détails que possible sur le problème et les étapes nécessaires pour le reproduire", + "support_warning_document": "Veuillez préciser quel type de document est à l'origine du problème et fournir la référence du document ou un lien", + "support_warning_drives": "Veuillez noter que les administrateurs ne sont pas en mesure d'identifier des dossiers ou documents à partir de leur nom. Pour les dossiers partagés, veuillez fournir la référence du dossier", + "support_warning_account": "Veuillez noter que les administrateurs ne sont pas en mesure de réinitialiser les mots de passe. Si vous avez perdu vos identifiants mais que vous êtes encore connecté, vous pouvez transférer vos données vers un nouveau compte", + "support_warning_prompt": "Merci de choisir la catégorie la plus pertinente pour qualifier votre problème, ceci aide les administrateurs à faire le tri et fournit des suggestions sur les informations à fournir", + "admin_jurisdictionTitle": "Pays d'hébergement", + "ui_saved": "{0} enregistré" }