From c53675c9d5e4803850806b943679e19a276666ce Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 6 Apr 2020 10:28:26 -0400 Subject: [PATCH 01/38] WIP worker rpc call queue --- lib/workers/index.js | 117 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 95 insertions(+), 22 deletions(-) diff --git a/lib/workers/index.js b/lib/workers/index.js index 7d8c1dcc7..847081104 100644 --- a/lib/workers/index.js +++ b/lib/workers/index.js @@ -124,18 +124,58 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { return response.expected(id)? guid(): id; }; - var workerIndex = 0; + var workerOffset = -1; + var getAvailableWorkerIndex = function () { + var L = workers.length; + if (L === 0) { + console.log("no workers available"); + return -1; + } + + // cycle through the workers once + // start from a different offset each time + // return -1 if none are available + + workerOffset = (workerOffset + 1) % L; + + var temp; + for (let i = 0; i < L; i++) { + temp = (workerOffset + i) % L; + if (workers[temp] && workers[temp].count > 0) { + return temp; + } + } + return -1; + }; + + var queue = []; + var MAX_JOBS = 32; //1; //8; + var sendCommand = function (msg, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); + var index = getAvailableWorkerIndex(); + + var state = workers[index]; + // if there is no worker available: + if (!isWorker(state)) { + console.log("queueing for later"); + // queue the message for when one becomes available + queue.push({ + msg: msg, + cb: _cb, + }); + return; + //return void cb("NO_WORKERS"); + } else { + console.log("worker #%s handling %s messages currently", index, MAX_JOBS + 1 - state.count); - workerIndex = (workerIndex + 1) % workers.length; - if (!isWorker(workers[workerIndex])) { - return void cb("NO_WORKERS"); } - var state = workers[workerIndex]; + console.log("%s queued messages", queue.length); - // XXX insert a queue here to prevent timeouts + console.log("[%s]\n", msg.command); + //console.log(msg); + + var cb = Util.once(Util.mkAsync(_cb)); const txid = guid(); msg.txid = txid; @@ -143,20 +183,67 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { // track which worker is doing which jobs state.tasks[txid] = msg; + state.count--; + + if (state.count < 0) { + console.log(state); + throw new Error("too many jobs"); // XXX + } + response.expect(txid, function (err, value) { // clean up when you get a response delete state[txid]; + state.count++; cb(err, value); }, 60000); state.worker.send(msg); }; + var backlogged; + var handleResponse = function (res) { + if (!res) { return; } + // handle log messages before checking if it was addressed to your PID + // it might still be useful to know what happened inside an orphaned worker + if (res.log) { + return void handleLog(res.log, res.label, res.info); + } + // but don't bother handling things addressed to other processes + // since it's basically guaranteed not to work + if (res.pid !== PID) { + return void Log.error("WRONG_PID", res); + } + + setTimeout(function () { + response.handle(res.txid, [res.error, res.value]); + + if (!queue.length) { + if (backlogged) { + backlogged = false; + console.log("queue has been drained"); + } + return; + } else { + backlogged = true; + console.log(queue, queue.length); + } + + console.log("taking queued message"); + + // XXX take a request from the queue + var nextMsg = queue.shift(); + sendCommand(nextMsg.msg, nextMsg.cb); // XXX doesn't feel right + console.log("%s queued messages remaining", queue.length); + + }, (Math.floor(Math.random() * 150) * 10)); + }; + const initWorker = function (worker, cb) { const txid = guid(); const state = { worker: worker, tasks: {}, + count: MAX_JOBS, //1, // XXX }; response.expect(txid, function (err) { @@ -171,21 +258,7 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { config: config, }); - worker.on('message', function (res) { - if (!res) { return; } - // handle log messages before checking if it was addressed to your PID - // it might still be useful to know what happened inside an orphaned worker - if (res.log) { - return void handleLog(res.log, res.label, res.info); - } - // but don't bother handling things addressed to other processes - // since it's basically guaranteed not to work - if (res.pid !== PID) { - return void Log.error("WRONG_PID", res); - } - - response.handle(res.txid, [res.error, res.value]); - }); + worker.on('message', handleResponse); var substituteWorker = Util.once(function () { Env.Log.info("SUBSTITUTE_DB_WORKER", ''); From dca2707ae3ae39aa82f07c04ad52e0f212bf8db9 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 7 Apr 2020 09:45:21 -0400 Subject: [PATCH 02/38] guard against incorrect types in /auth/ --- www/auth/main.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/www/auth/main.js b/www/auth/main.js index c155204ec..fcbeaf3af 100644 --- a/www/auth/main.js +++ b/www/auth/main.js @@ -71,7 +71,7 @@ define([ // Get contacts and extract their avatar channel and key var getData = function (obj, href) { var parsed = Hash.parsePadUrl(href); - if (!parsed || parsed.type !== "file") { return; } // XXX + if (!parsed || parsed.type !== "file") { return; } var secret = Hash.getSecrets('file', parsed.hash); if (!secret.keys || !secret.channel) { return; } obj.avatarKey = Hash.encodeBase64(secret.keys && secret.keys.cryptKey); @@ -81,7 +81,7 @@ define([ contacts.friends = proxy.friends || {}; Object.keys(contacts.friends).map(function (key) { var friend = contacts.friends[key]; - // if (!friend) { return; } // XXX how should this be handled? + if (!friend) { return; } var ret = { edPublic: friend.edPublic, name: friend.displayName, @@ -91,7 +91,7 @@ define([ }); Object.keys(contacts.teams).map(function (key) { var team = contacts.teams[key]; - // if (!team) { return; } // XXX how should this be handled. Is this possible? + if (!team) { return; } var avatar = team.metadata && team.metadata.avatar; var ret = { edPublic: team.keys && team.keys.drive && team.keys.drive.edPublic, From 95965c1deec3a75d506834374f49ed5bbacebee5 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 7 Apr 2020 20:03:41 -0400 Subject: [PATCH 03/38] keep a parallel implementation of the pin loader to validate the new one --- lib/pins.js | 63 ++++++++++++++++++++++++++++++++++ scripts/compare-pin-methods.js | 42 +++++++++++++++++++++++ scripts/evict-inactive.js | 2 +- 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 scripts/compare-pin-methods.js diff --git a/lib/pins.js b/lib/pins.js index 41e871446..d840991a3 100644 --- a/lib/pins.js +++ b/lib/pins.js @@ -7,6 +7,9 @@ const Path = require("path"); const Util = require("./common-util"); const Plan = require("./plan"); +const Semaphore = require('saferphore'); +const nThen = require('nthen'); + /* Accepts a reference to an object, and... either a string describing which log is being processed (backwards compatibility), or a function which will log the error with all relevant data @@ -194,3 +197,63 @@ Pins.list = function (_done, config) { }).start(); }); }; + +Pins.load = function (cb, config) { + const sema = Semaphore.create(config.workers || 5); + + let dirList; + const fileList = []; + const pinned = {}; + + var pinPath = config.pinPath || './pins'; + var done = Util.once(cb); + + nThen((waitFor) => { + // recurse over the configured pinPath, or the default + Fs.readdir(pinPath, waitFor((err, list) => { + if (err) { + if (err.code === 'ENOENT') { + dirList = []; + return; // this ends up calling back with an empty object + } + waitFor.abort(); + return void done(err); + } + dirList = list; + })); + }).nThen((waitFor) => { + dirList.forEach((f) => { + sema.take((returnAfter) => { + // iterate over all the subdirectories in the pin store + Fs.readdir(Path.join(pinPath, f), waitFor(returnAfter((err, list2) => { + if (err) { + waitFor.abort(); + return void done(err); + } + list2.forEach((ff) => { + if (config && config.exclude && config.exclude.indexOf(ff) > -1) { return; } + fileList.push(Path.join(pinPath, f, ff)); + }); + }))); + }); + }); + }).nThen((waitFor) => { + fileList.forEach((f) => { + sema.take((returnAfter) => { + Fs.readFile(f, waitFor(returnAfter((err, content) => { + if (err) { + waitFor.abort(); + return void done(err); + } + const hashes = Pins.calculateFromLog(content.toString('utf8'), f); + hashes.forEach((x) => { + (pinned[x] = pinned[x] || {})[f.replace(/.*\/([^/]*).ndjson$/, (x, y)=>y)] = 1; + }); + }))); + }); + }); + }).nThen(() => { + done(void 0, pinned); + }); +}; + diff --git a/scripts/compare-pin-methods.js b/scripts/compare-pin-methods.js new file mode 100644 index 000000000..de7ef114d --- /dev/null +++ b/scripts/compare-pin-methods.js @@ -0,0 +1,42 @@ +/* jshint esversion: 6, node: true */ +const nThen = require("nthen"); +const Pins = require("../lib/pins"); +const Assert = require("assert"); + +const config = require("../lib/load-config"); + +var compare = function () { + console.log(config); + var conf = { + pinPath: config.pinPath, + }; + + var list, load; + + nThen(function (w) { + Pins.list(w(function (err, p) { + if (err) { throw err; } + list = p; + console.log(p); + console.log(list); + console.log(); + }), conf); + }).nThen(function (w) { + Pins.load(w(function (err, p) { + if (err) { throw err; } + load = p; + console.log(load); + console.log(); + }), conf); + }).nThen(function () { + console.log({ + listLength: Object.keys(list).length, + loadLength: Object.keys(load).length, + }); + + Assert.deepEqual(list, load); + console.log("methods are equivalent"); + }); +}; + +compare(); diff --git a/scripts/evict-inactive.js b/scripts/evict-inactive.js index a3a595ca4..1d7b87e91 100644 --- a/scripts/evict-inactive.js +++ b/scripts/evict-inactive.js @@ -42,7 +42,7 @@ nThen(function (w) { store = _; })); // load the list of pinned files so you know which files // should not be archived or deleted - Pins.list(w(function (err, _) { + Pins.load(w(function (err, _) { if (err) { w.abort(); return void console.error(err); From 7fac997e93236312ddb68471b858d5242b060044 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 8 Apr 2020 09:38:48 -0400 Subject: [PATCH 04/38] increase some file storage timeouts related to streams --- lib/storage/file.js | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/lib/storage/file.js b/lib/storage/file.js index 2d27ce185..00222c3f6 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -14,6 +14,28 @@ const readFileBin = require("../stream-file").readFileBin; const BatchRead = require("../batch-read"); const Schedule = require("../schedule"); + +/* Each time you write to a channel it will either use an open file descriptor + for that channel or open a new descriptor if one is not available. These are + automatically closed after this window to prevent a file descriptor leak, so + writes that take longer than this time may be dropped! */ +const CHANNEL_WRITE_WINDOW = 300000; + +/* Each time you read a channel it will have this many milliseconds to complete + otherwise it will be closed to prevent a file descriptor leak. The server will + lock up if it uses all available file descriptors, so it's important to close + them. The tradeoff with this timeout is that some functions, the stream, and + and the timeout itself are stored in memory. A longer timeout uses more memory + and running out of memory will also kill the server. */ +const STREAM_CLOSE_TIMEOUT = 300000; + +/* The above timeout closes the stream, but apparently that doesn't always work. + We set yet another timeout to allow the runtime to gracefully close the stream + (flushing all pending writes/reads and doing who knows what else). After this timeout + it will be MERCILESSLY DESTROYED. This isn't graceful, but again, file descriptor + leaks are bad. */ +const STREAM_DESTROY_TIMEOUT = 30000; + const isValidChannelId = function (id) { return typeof(id) === 'string' && id.length >= 32 && id.length < 50 && @@ -64,7 +86,7 @@ const destroyStream = function (stream) { try { stream.close(); } catch (err) { console.error(err); } setTimeout(function () { try { stream.destroy(); } catch (err) { console.error(err); } - }, 15000); + }, STREAM_DESTROY_TIMEOUT); }; const ensureStreamCloses = function (stream, id, ms) { @@ -74,7 +96,7 @@ const ensureStreamCloses = function (stream, id, ms) { // this can only be a timeout error... console.log("stream close error:", err, id); } - }), ms || 45000), []); + }), ms || STREAM_CLOSE_TIMEOUT), []); }; // readMessagesBin asynchronously iterates over the messages in a channel log @@ -729,7 +751,7 @@ var getChannel = function (env, id, _callback) { delete env.channels[id]; destroyStream(channel.writeStream, path); //console.log("closing writestream"); - }, 120000); + }, CHANNEL_WRITE_WINDOW); channel.delayClose(); env.channels[id] = channel; done(void 0, channel); From bef18a93201adb4bd067551504616f7407180865 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 8 Apr 2020 10:28:26 -0400 Subject: [PATCH 05/38] hotfix --- lib/commands/pin-rpc.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/commands/pin-rpc.js b/lib/commands/pin-rpc.js index f3ade0489..258c6f53b 100644 --- a/lib/commands/pin-rpc.js +++ b/lib/commands/pin-rpc.js @@ -56,9 +56,13 @@ var loadUserPins = function (Env, safeKey, cb) { }; var truthyKeys = function (O) { - return Object.keys(O).filter(function (k) { - return O[k]; - }); + try { + return Object.keys(O).filter(function (k) { + return O[k]; + }); + } catch (err) { + return []; + } }; var getChannelList = Pinning.getChannelList = function (Env, safeKey, _cb) { From bd3e18d1a87d93caf63faeca7e81f3bfe0527758 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 8 Apr 2020 12:08:26 -0400 Subject: [PATCH 06/38] only cache a user's pinned channels if there are no errors when loading them --- lib/commands/pin-rpc.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/commands/pin-rpc.js b/lib/commands/pin-rpc.js index 258c6f53b..a6f1642c6 100644 --- a/lib/commands/pin-rpc.js +++ b/lib/commands/pin-rpc.js @@ -49,7 +49,6 @@ var loadUserPins = function (Env, safeKey, cb) { // only put this into the cache if it completes session.channels = value; } - session.channels = value; done(value); }); }); From ebd4998edd2ba024daf146503f118ff8da9e9875 Mon Sep 17 00:00:00 2001 From: Weblate Date: Thu, 9 Apr 2020 19:08:12 +0200 Subject: [PATCH 07/38] Translated using Weblate (Catalan) Currently translated at 50.8% (633 of 1246 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/ca/ --- www/common/translations/messages.ca.json | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/www/common/translations/messages.ca.json b/www/common/translations/messages.ca.json index aca3b8068..a10a9fffd 100644 --- a/www/common/translations/messages.ca.json +++ b/www/common/translations/messages.ca.json @@ -350,7 +350,7 @@ "fm_info_root": "Creeu aquí tantes carpetes imbrincades com vulgueu per ordenar els vostres fitxers.", "fm_info_unsorted": "Conté tots els fitxers que heu visitat i que encara no estan ordenats, a \"Documents\" o desplaçats a la \"Paperera\".", "fm_info_template": "Conté tots els documents desats com plantilles i que podeu reutilitzar quan vulgueu crear un nou document.", - "fm_info_recent": "Llista els documents modificats o oberts recentment.", + "fm_info_recent": "Aquests documents s'han modificat o obert darrerament, per vós o per alguna persona col·laboradora.", "fm_info_trash": "Buideu la paperera per alliberar espai al vostre CryptDrive.", "fm_info_allFiles": "Conté tots els fitxers de \"Documents\", \"Desordenats\" i \"Paperera\". No podeu desplaçar o suprimir fitxers des d'aquí.", "fm_info_anonymous": "No heu iniciat la sessió, per tant, els vostres documents caducaran d'aquí a 3 mesos (saber-ne més). Es desen al vostre navegador, per tant, si netegeu el vostre historial podríeu perdre'ls.
Registreu-vos o Inicieu la sessió per mantenir-los accessibles.
", @@ -515,7 +515,7 @@ "settings_pinningError": "Alguna cosa no ha funcionat correctament", "settings_usageAmount": "Els vostres documents fixats ocupen {0} MB", "settings_logoutEverywhereButton": "Tanca la sessió", - "settings_logoutEverywhereTitle": "Tanca la sessió arreu", + "settings_logoutEverywhereTitle": "Tanca les sessions remotes", "settings_logoutEverywhere": "Tanca totes les altres sessions", "settings_logoutEverywhereConfirm": "De debò? Haureu de tornar a iniciar la vostra sessió a tots els dispositius.", "settings_driveDuplicateTitle": "Documents propis duplicats", @@ -573,7 +573,7 @@ "upload_success": "El fitxer ({0}) ha estat carregat correctament i afegit al vostre CryptDrive.", "upload_notEnoughSpace": "No hi ha prou espai al CryptDrive per aquest fitxer.", "upload_notEnoughSpaceBrief": "No hi ha prou espai", - "upload_tooLarge": "Aquest fitxer supera la mida màxima permesa.", + "upload_tooLarge": "Aquest fitxer supera la mida màxima permesa pel vostre compte", "upload_tooLargeBrief": "El fitxer és massa gran", "upload_choose": "Trieu un fitxer", "upload_pending": "Pendent", @@ -619,5 +619,20 @@ "download_resourceNotAvailable": "El recurs sol·licitat no estava disponible... Premeu Esc per continuar.", "about_contributors": "Col·laboracions clau", "about_core": "Desenvolupament principal", - "about_intro": "CryptPad s'ha creat dins l'Equip de Recerca de XWiki SAS, una petita empresa de París, França i Iasi, Romania. Hi ha 3 membres de l'equip central treballant amb CryptPad més una quantitat de persones col·laboradores, dins i fora d'XWiki SAS." + "about_intro": "CryptPad s'ha creat dins l'Equip de Recerca de XWiki SAS, una petita empresa de París, França i Iasi, Romania. Hi ha 3 membres de l'equip central treballant amb CryptPad més una quantitat de persones col·laboradores, dins i fora d'XWiki SAS.", + "main_catch_phrase": "El Núvol Coneixement Zero", + "main_footerText": "Amb Cryptpad, podeu crear documents col·laboratius per prendre notes i posar en ordre idees de forma conjunta de forma ràpida.", + "footer_applications": "Aplicacions", + "footer_contact": "Contacte", + "footer_aboutUs": "Sobre nosaltres", + "about": "Sobre", + "contact": "Contacte", + "blog": "Bloc", + "topbar_whatIsCryptpad": "Què és CryptPad", + "whatis_collaboration": "Col·laboració fàcil i ràpida", + "whatis_title": "Què és CryptPad", + "terms": "Condicions d'ús", + "main_info": "

Col·laboreu amb Confiança

\nFeu créixer les vostres idees conjuntament amb documents compartits mentre la tecnologia Coneixement Zero assegura la vostra privacitat; fins i tot per nosaltres.", + "whatis_collaboration_p1": "Amb CryptPad, podeu crear de forma ràpida, documents col·laboratius per prendre notes i posar en ordre idees conjuntament. Quan us registreu i inicieu la vostra sessió, teniu la capacitat de carregar fitxers i un CryptDrive on podeu organitzar tots els vostres documents. Com a persona registrada disposeu de 50MB d'espai gratuït.", + "privacy": "Privacitat" } From e527867e2ea763d674803768fab65f644f39dcf6 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 9 Apr 2020 16:29:52 -0400 Subject: [PATCH 08/38] queue blob and channel deletions per-user --- lib/commands/channel.js | 28 +++++++++++++++++++--------- lib/historyKeeper.js | 1 + 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/lib/commands/channel.js b/lib/commands/channel.js index d35e15241..fce5b048c 100644 --- a/lib/commands/channel.js +++ b/lib/commands/channel.js @@ -54,16 +54,8 @@ Channel.clearOwnedChannel = function (Env, safeKey, channelId, cb, Server) { }); }; -Channel.removeOwnedChannel = function (Env, safeKey, channelId, cb, Server) { - if (typeof(channelId) !== 'string' || !Core.isValidId(channelId)) { - return cb('INVALID_ARGUMENTS'); - } +var archiveOwnedChannel = function (Env, safeKey, channelId, cb, Server) { var unsafeKey = Util.unescapeKeyCharacters(safeKey); - - if (Env.blobStore.isFileId(channelId)) { - return void Env.removeOwnedBlob(channelId, safeKey, cb); - } - Metadata.getMetadata(Env, channelId, function (err, metadata) { if (err) { return void cb(err); } if (!Core.hasOwners(metadata)) { return void cb('E_NO_OWNERS'); } @@ -124,6 +116,24 @@ Channel.removeOwnedChannel = function (Env, safeKey, channelId, cb, Server) { }); }; +Channel.removeOwnedChannel = function (Env, safeKey, channelId, __cb, Server) { + var _cb = Util.once(Util.mkAsync(__cb)); + + if (typeof(channelId) !== 'string' || !Core.isValidId(channelId)) { + return _cb('INVALID_ARGUMENTS'); + } + + // archiving large channels or files can be expensive, so do it one at a time + // for any given user to ensure that nobody can use too much of the server's resources + Env.queueDeletes(safeKey, function (next) { + var cb = Util.both(_cb, next); + if (Env.blobStore.isFileId(channelId)) { + return void Env.removeOwnedBlob(channelId, safeKey, cb); + } + archiveOwnedChannel(Env, safeKey, channelId, cb, Server); + }); +}; + Channel.trimHistory = function (Env, safeKey, data, cb) { if (!(data && typeof(data.channel) === 'string' && typeof(data.hash) === 'string' && data.hash.length === 64)) { return void cb('INVALID_ARGS'); diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index f70dba006..2b5ec8770 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -38,6 +38,7 @@ module.exports.create = function (config, cb) { metadata_cache: {}, channel_cache: {}, queueStorage: WriteQueue(), + queueDeletes: WriteQueue(), batchIndexReads: BatchRead("HK_GET_INDEX"), batchMetadata: BatchRead('GET_METADATA'), From 09bf0a54d81ca2fe651d22df0fc60c8c5b1c2d4a Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 9 Apr 2020 16:30:22 -0400 Subject: [PATCH 09/38] add access control header --- docs/example.nginx.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/example.nginx.conf b/docs/example.nginx.conf index ea8224c14..f6e163910 100644 --- a/docs/example.nginx.conf +++ b/docs/example.nginx.conf @@ -54,6 +54,7 @@ server { add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; add_header X-XSS-Protection "1; mode=block"; add_header X-Content-Type-Options nosniff; + add_header Access-Control-Allow-Origin "*"; # add_header X-Frame-Options "SAMEORIGIN"; # Insert the path to your CryptPad repository root here From de6594f4c9a1155d0c7ee0b2aead713f99b20930 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 9 Apr 2020 16:31:16 -0400 Subject: [PATCH 10/38] add some XXX notes --- www/common/onlyoffice/inner.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/www/common/onlyoffice/inner.js b/www/common/onlyoffice/inner.js index 5ac3da269..3a824214c 100644 --- a/www/common/onlyoffice/inner.js +++ b/www/common/onlyoffice/inner.js @@ -1091,12 +1091,12 @@ define([ var x2tSaveAndConvertData = function(data, filename, extension, finalFilename) { // Perform the x2t conversion - require(['/common/onlyoffice/x2t/x2t.js'], function() { + require(['/common/onlyoffice/x2t/x2t.js'], function() { // XXX why does this fail without an access-control-allow-origin header? var x2t = window.Module; x2t.run(); if (x2tInitialized) { debug("x2t runtime already initialized"); - x2tSaveAndConvertDataInternal(x2t, data, filename, extension, finalFilename); + x2tSaveAndConvertDataInternal(x2t, data, filename, extension, finalFilename); // XXX shouldn't this return ? } x2t.onRuntimeInitialized = function() { From e3f42cbc20ecec8b4e0ed8d453d12ae31197a1d6 Mon Sep 17 00:00:00 2001 From: Martin Alexander Neumann Date: Sat, 11 Apr 2020 20:45:28 +0200 Subject: [PATCH 11/38] Allow config.defaultStorageLimit to be zero bytes --- lib/historyKeeper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index f70dba006..1fc2a2f2f 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -98,7 +98,7 @@ module.exports.create = function (config, cb) { paths.staging = keyOrDefaultString('blobStagingPath', './blobstage'); paths.blob = keyOrDefaultString('blobPath', './blob'); - Env.defaultStorageLimit = typeof(config.defaultStorageLimit) === 'number' && config.defaultStorageLimit > 0? + Env.defaultStorageLimit = typeof(config.defaultStorageLimit) === 'number' && config.defaultStorageLimit >= 0? config.defaultStorageLimit: Core.DEFAULT_LIMIT; From 5fe7f55f51ad0da26ecc9681098e45d351c0dd2d Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 14 Apr 2020 10:47:53 +0200 Subject: [PATCH 12/38] Fix share with a team when we don't have contacts yet --- www/common/common-ui-elements.js | 88 ++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 37 deletions(-) diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index a9c682afc..b7c6310f5 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -175,9 +175,9 @@ define([ var common = config.common; var sframeChan = common.getSframeChannel(); var title = config.title; - var friends = config.friends; + var friends = config.friends || {}; + var teams = config.teams || {}; var myName = common.getMetadataMgr().getUserData().name; - if (!friends) { return; } var order = []; var smallCurves = Object.keys(friends).map(function (c) { @@ -206,40 +206,28 @@ define([ delete friends[curve]; }); - var friendsList = UIElements.getUserGrid(Messages.share_linkFriends, { - common: common, - data: friends, - noFilter: false, - large: true - }, refreshButtons); - var friendDiv = friendsList.div; - $div.append(friendDiv); - var others = friendsList.icons; + var others = []; + if (Object.keys(friends).length) { + var friendsList = UIElements.getUserGrid(Messages.share_linkFriends, { + common: common, + data: friends, + noFilter: false, + large: true + }, refreshButtons); + var friendDiv = friendsList.div; + $div.append(friendDiv); + others = friendsList.icons; + } - var privateData = common.getMetadataMgr().getPrivateData(); - var teamsData = Util.tryParse(JSON.stringify(privateData.teams)) || {}; - var teams = {}; - Object.keys(teamsData).forEach(function (id) { - // config.teamId only exists when we're trying to share a pad from a team drive - // In this case, we don't want to share the pad with the current team - if (config.teamId && config.teamId === id) { return; } - if (!teamsData[id].hasSecondaryKey) { return; } - var t = teamsData[id]; - teams[t.edPublic] = { - notifications: true, - displayName: t.name, - edPublic: t.edPublic, - avatar: t.avatar, - id: id - }; - }); - var teamsList = UIElements.getUserGrid(Messages.share_linkTeam, { - common: common, - noFilter: true, - large: true, - data: teams - }, refreshButtons); - $div.append(teamsList.div); + if (Object.keys(teams).length) { + var teamsList = UIElements.getUserGrid(Messages.share_linkTeam, { + common: common, + noFilter: true, + large: true, + data: teams + }, refreshButtons); + $div.append(teamsList.div); + } var shareButton = { className: 'primary cp-share-with-friends', @@ -395,6 +383,26 @@ define([ } }; + var getEditableTeams = function (common, config) { + var privateData = common.getMetadataMgr().getPrivateData(); + var teamsData = Util.tryParse(JSON.stringify(privateData.teams)) || {}; + var teams = {}; + Object.keys(teamsData).forEach(function (id) { + // config.teamId only exists when we're trying to share a pad from a team drive + // In this case, we don't want to share the pad with the current team + if (config.teamId && config.teamId === id) { return; } + if (!teamsData[id].hasSecondaryKey) { return; } + var t = teamsData[id]; + teams[t.edPublic] = { + notifications: true, + displayName: t.name, + edPublic: t.edPublic, + avatar: t.avatar, + id: id + }; + }); + return teams; + }; var makeBurnAfterReadingUrl = function (common, href, channel, cb) { var keyPair = Hash.generateSignPair(); var parsed = Hash.parsePadUrl(href); @@ -643,7 +651,10 @@ define([ // Share with contacts tab - var hasFriends = Object.keys(config.friends || {}).length !== 0; + var teams = getEditableTeams(common, config); + config.teams = teams; + var hasFriends = Object.keys(config.friends || {}).length || + Object.keys(teams).length; var onFriendShare = Util.mkEvent(); @@ -926,7 +937,10 @@ define([ }); // share with contacts tab - var hasFriends = Object.keys(config.friends || {}).length !== 0; + var teams = getEditableTeams(common, config); + config.teams = teams; + var hasFriends = Object.keys(config.friends || {}).length || + Object.keys(teams).length; var friendsObject = hasFriends ? createShareWithFriends(config, null, getLinkValue) : noContactsMessage(common); var friendsList = friendsObject.content; From ca2dc85b3407d63d5fc4b24e55e557ef02479a0c Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 14 Apr 2020 15:17:41 -0400 Subject: [PATCH 13/38] reduce eye-bleed with gentler styles for mermaid charts --- www/common/diffMarked.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index f53828e17..0f0e4a092 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -23,8 +23,17 @@ define([ init: function () {} }; + var mermaidThemeCSS = ".node rect { fill: #DDD; stroke: #AAA; } " + + "rect.task, rect.task0, rect.task2 { stroke-width: 1 !important; rx: 0 !important; } " + + "g.grid g.tick line { opacity: 0.25; }" + + "g.today line { stroke: red; stroke-width: 1; stroke-dasharray: 3; opacity: 0.5; }"; + require(['mermaid', 'css!/code/mermaid-new.css'], function (_Mermaid) { Mermaid = _Mermaid; + Mermaid.initialize({ + gantt: { axisFormat: '%m-%d', }, + "themeCSS": mermaidThemeCSS, + }); }); var highlighter = function () { From 9a7681dc5d6f157e8a99af324cd32c29ddf7474e Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 14 Apr 2020 15:18:20 -0400 Subject: [PATCH 14/38] add a note about handling syntax errors in mermaid --- www/common/diffMarked.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index 0f0e4a092..c964ee63e 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -360,6 +360,12 @@ define([ // retrieve the attached source code which it was drawn var src = el.getAttribute('mermaid-source'); +/* The new source might have syntax errors that will prevent rendering. + It might be preferable to keep the existing state instead of removing it + if you don't have something better to display. Ideally we should display + the cause of the syntax error so that the user knows what to correct. */ + //if (!Mermaid.parse(src)) { } // TODO + // check if that source exists in the set of charts which are about to be rendered if (mermaid_source.indexOf(src) === -1) { // if it's not, then you can remove it From 74567e0cf4e90f7abaa5e75ea647e965e3bd331f Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 14 Apr 2020 15:53:58 -0400 Subject: [PATCH 15/38] return process memory usage in GET CACHE STATS RPC --- lib/commands/admin-rpc.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index ed1d5b8a4..db2413518 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -50,6 +50,7 @@ var getCacheStats = function (env, server, cb) { metaSize: metaSize, channel: channelCount, channelSize: channelSize, + memoryUsage: process.memoryUsage(), }); }; From f83179cb62e13d06d6288fd4e185106f0dab0ba9 Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 15 Apr 2020 16:07:18 +0200 Subject: [PATCH 16/38] Fix mediatag cache preventing preview modal --- www/common/diffMarked.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index f53828e17..45d426124 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -418,7 +418,7 @@ define([ throw new Error(patch); } else { DD.apply($content[0], patch); - var $mts = $content.find('media-tag:not(:has(*))'); + var $mts = $content.find('media-tag'); $mts.each(function (i, el) { var $mt = $(el).contextmenu(function (e) { e.preventDefault(); @@ -426,6 +426,16 @@ define([ $(contextMenu.menu).find('li').show(); contextMenu.show(e); }); + if ($mt.children().length) { + $mt.off('dblclick preview'); + $mt.on('preview', onPreview($mt)); + if ($mt.find('img').length) { + $mt.on('dblclick', function () { + $mt.trigger('preview'); + }); + } + return; + } MediaTag(el); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(mutation) { From 9ed26cfeb0bd83afc5d73d297ae7af310430bf19 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 15 Apr 2020 12:40:18 -0400 Subject: [PATCH 17/38] lint compliance --- lib/commands/admin-rpc.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index db2413518..c60cdcfbc 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -1,4 +1,5 @@ /*jshint esversion: 6 */ +/* globals process */ const nThen = require("nthen"); const getFolderSize = require("get-folder-size"); const Util = require("../common-util"); From e8b1fcf710c69d701d0bf73f49d5f7f0ebe06169 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 15 Apr 2020 13:59:54 -0400 Subject: [PATCH 18/38] solve some cases where crashing workers could result in an invalid state for the queue --- lib/workers/index.js | 83 +++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 51 deletions(-) diff --git a/lib/workers/index.js b/lib/workers/index.js index 847081104..d2944225c 100644 --- a/lib/workers/index.js +++ b/lib/workers/index.js @@ -124,11 +124,12 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { return response.expected(id)? guid(): id; }; + const MAX_JOBS = 32; var workerOffset = -1; var getAvailableWorkerIndex = function () { var L = workers.length; if (L === 0) { - console.log("no workers available"); + console.log("no workers available"); // XXX return -1; } @@ -141,7 +142,15 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { var temp; for (let i = 0; i < L; i++) { temp = (workerOffset + i) % L; - if (workers[temp] && workers[temp].count > 0) { +/* I'd like for this condition to be more efficient + (`Object.keys` is sub-optimal) but I found some bugs in my initial + implementation stemming from a task counter variable going out-of-sync + with reality when a worker crashed and its tasks were re-assigned to + its substitute. I'm sure it can be done correctly and efficiently, + but this is a relatively easy way to make sure it's always up to date. + We'll see how it performs in practice before optimizing. +*/ + if (workers[temp] && Object.keys(workers[temp]).length < MAX_JOBS) { return temp; } } @@ -149,7 +158,6 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { }; var queue = []; - var MAX_JOBS = 32; //1; //8; var sendCommand = function (msg, _cb) { var index = getAvailableWorkerIndex(); @@ -157,24 +165,14 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { var state = workers[index]; // if there is no worker available: if (!isWorker(state)) { - console.log("queueing for later"); // queue the message for when one becomes available queue.push({ msg: msg, cb: _cb, }); return; - //return void cb("NO_WORKERS"); - } else { - console.log("worker #%s handling %s messages currently", index, MAX_JOBS + 1 - state.count); - } - console.log("%s queued messages", queue.length); - - console.log("[%s]\n", msg.command); - //console.log(msg); - var cb = Util.once(Util.mkAsync(_cb)); const txid = guid(); @@ -183,24 +181,12 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { // track which worker is doing which jobs state.tasks[txid] = msg; - state.count--; - if (state.count < 0) { - console.log(state); - throw new Error("too many jobs"); // XXX - } - - response.expect(txid, function (err, value) { - // clean up when you get a response - delete state[txid]; - state.count++; - cb(err, value); - }, 60000); + response.expect(txid, cb, 60000); state.worker.send(msg); }; - var backlogged; - var handleResponse = function (res) { + var handleResponse = function (state, res) { if (!res) { return; } // handle log messages before checking if it was addressed to your PID // it might still be useful to know what happened inside an orphaned worker @@ -213,28 +199,22 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { return void Log.error("WRONG_PID", res); } - setTimeout(function () { - response.handle(res.txid, [res.error, res.value]); - - if (!queue.length) { - if (backlogged) { - backlogged = false; - console.log("queue has been drained"); - } - return; - } else { - backlogged = true; - console.log(queue, queue.length); - } - - console.log("taking queued message"); - - // XXX take a request from the queue - var nextMsg = queue.shift(); - sendCommand(nextMsg.msg, nextMsg.cb); // XXX doesn't feel right - console.log("%s queued messages remaining", queue.length); - - }, (Math.floor(Math.random() * 150) * 10)); + if (!res.txid) { return; } + response.handle(res.txid, [res.error, res.value]); + delete state.tasks[res.txid]; + if (!queue.length) { return; } + + var nextMsg = queue.shift(); +/* `nextMsg` was at the top of the queue. + We know that a job just finished and all of this code + is synchronous, so calling `sendCommand` should take the worker + which was just freed up. This is somewhat fragile though, so + be careful if you want to modify this block. The risk is that + we take something that was at the top of the queue and push it + to the back because the following msg took its place. OR, in an + even worse scenario, we cycle through the queue but don't run anything. +*/ + sendCommand(nextMsg.msg, nextMsg.cb); }; const initWorker = function (worker, cb) { @@ -243,7 +223,6 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { const state = { worker: worker, tasks: {}, - count: MAX_JOBS, //1, // XXX }; response.expect(txid, function (err) { @@ -258,7 +237,9 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { config: config, }); - worker.on('message', handleResponse); + worker.on('message', function (res) { + handleResponse(state, res); + }); var substituteWorker = Util.once(function () { Env.Log.info("SUBSTITUTE_DB_WORKER", ''); From e0a6852b7979b8af50d3d109ac80dc4ee3457666 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 16 Apr 2020 10:45:18 -0400 Subject: [PATCH 19/38] correctly serialize an error log --- lib/hk-util.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/hk-util.js b/lib/hk-util.js index 0bb96a52b..b28cad497 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -621,7 +621,10 @@ const handleGetHistory = function (Env, Server, seq, userId, parsed) { Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(msg)], readMore); }, (err) => { if (err && err.code !== 'ENOENT') { - if (err.message !== 'EINVAL') { Log.error("HK_GET_HISTORY", err); } + if (err.message !== 'EINVAL') { Log.error("HK_GET_HISTORY", { + err: err && err.message, + stack: err && err.stack, + }); } const parsedMsg = {error:err.message, channel: channelName, txid: txid}; Server.send(userId, [0, HISTORY_KEEPER_ID, 'MSG', userId, JSON.stringify(parsedMsg)]); return; From 04ab7f538aae6887d536f1e4b10b0af3a4262280 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 16 Apr 2020 13:02:07 -0400 Subject: [PATCH 20/38] recover from worker faults and unify worker types --- lib/workers/crypto-worker.js | 113 ----------------------------- lib/workers/db-worker.js | 86 ++++++++++++++++++++++ lib/workers/index.js | 135 ++++++++++------------------------- 3 files changed, 122 insertions(+), 212 deletions(-) delete mode 100644 lib/workers/crypto-worker.js diff --git a/lib/workers/crypto-worker.js b/lib/workers/crypto-worker.js deleted file mode 100644 index 5ed58ac7c..000000000 --- a/lib/workers/crypto-worker.js +++ /dev/null @@ -1,113 +0,0 @@ -/* jshint esversion: 6 */ -/* global process */ -const Nacl = require('tweetnacl/nacl-fast'); - -const COMMANDS = {}; - -COMMANDS.INLINE = function (data, cb) { - var signedMsg; - try { - signedMsg = Nacl.util.decodeBase64(data.msg); - } catch (e) { - return void cb('E_BAD_MESSAGE'); - } - - var validateKey; - try { - validateKey = Nacl.util.decodeBase64(data.key); - } catch (e) { - return void cb("E_BADKEY"); - } - // validate the message - const validated = Nacl.sign.open(signedMsg, validateKey); - if (!validated) { - return void cb("FAILED"); - } - cb(); -}; - -const checkDetachedSignature = function (signedMsg, signature, publicKey) { - if (!(signedMsg && publicKey)) { return false; } - - var signedBuffer; - var pubBuffer; - var signatureBuffer; - - try { - signedBuffer = Nacl.util.decodeUTF8(signedMsg); - } catch (e) { - throw new Error("INVALID_SIGNED_BUFFER"); - } - - try { - pubBuffer = Nacl.util.decodeBase64(publicKey); - } catch (e) { - throw new Error("INVALID_PUBLIC_KEY"); - } - - try { - signatureBuffer = Nacl.util.decodeBase64(signature); - } catch (e) { - throw new Error("INVALID_SIGNATURE"); - } - - if (pubBuffer.length !== 32) { - throw new Error("INVALID_PUBLIC_KEY_LENGTH"); - } - - if (signatureBuffer.length !== 64) { - throw new Error("INVALID_SIGNATURE_LENGTH"); - } - - if (Nacl.sign.detached.verify(signedBuffer, signatureBuffer, pubBuffer) !== true) { - throw new Error("FAILED"); - } -}; - -COMMANDS.DETACHED = function (data, cb) { - try { - checkDetachedSignature(data.msg, data.sig, data.key); - } catch (err) { - return void cb(err && err.message); - } - cb(); -}; - -COMMANDS.HASH_CHANNEL_LIST = function (data, cb) { - var channels = data.channels; - if (!Array.isArray(channels)) { return void cb('INVALID_CHANNEL_LIST'); } - var uniques = []; - - channels.forEach(function (a) { - if (uniques.indexOf(a) === -1) { uniques.push(a); } - }); - uniques.sort(); - - var hash = Nacl.util.encodeBase64(Nacl.hash(Nacl - .util.decodeUTF8(JSON.stringify(uniques)))); - - cb(void 0, hash); -}; - -process.on('message', function (data) { - if (!data || !data.txid) { - return void process.send({ - error:'E_INVAL' - }); - } - - const cb = function (err, value) { - process.send({ - txid: data.txid, - error: err, - value: value, - }); - }; - - const command = COMMANDS[data.command]; - if (typeof(command) !== 'function') { - return void cb("E_BAD_COMMAND"); - } - - command(data, cb); -}); diff --git a/lib/workers/db-worker.js b/lib/workers/db-worker.js index 0b9de4f53..d76ce61dd 100644 --- a/lib/workers/db-worker.js +++ b/lib/workers/db-worker.js @@ -12,6 +12,7 @@ const Core = require("../commands/core"); const Saferphore = require("saferphore"); const Logger = require("../log"); const Tasks = require("../storage/tasks"); +const Nacl = require('tweetnacl/nacl-fast'); const Env = { Log: {}, @@ -432,6 +433,91 @@ const COMMANDS = { RUN_TASKS: runTasks, }; +COMMANDS.INLINE = function (data, cb) { + var signedMsg; + try { + signedMsg = Nacl.util.decodeBase64(data.msg); + } catch (e) { + return void cb('E_BAD_MESSAGE'); + } + + var validateKey; + try { + validateKey = Nacl.util.decodeBase64(data.key); + } catch (e) { + return void cb("E_BADKEY"); + } + // validate the message + const validated = Nacl.sign.open(signedMsg, validateKey); + if (!validated) { + return void cb("FAILED"); + } + cb(); +}; + +const checkDetachedSignature = function (signedMsg, signature, publicKey) { + if (!(signedMsg && publicKey)) { return false; } + + var signedBuffer; + var pubBuffer; + var signatureBuffer; + + try { + signedBuffer = Nacl.util.decodeUTF8(signedMsg); + } catch (e) { + throw new Error("INVALID_SIGNED_BUFFER"); + } + + try { + pubBuffer = Nacl.util.decodeBase64(publicKey); + } catch (e) { + throw new Error("INVALID_PUBLIC_KEY"); + } + + try { + signatureBuffer = Nacl.util.decodeBase64(signature); + } catch (e) { + throw new Error("INVALID_SIGNATURE"); + } + + if (pubBuffer.length !== 32) { + throw new Error("INVALID_PUBLIC_KEY_LENGTH"); + } + + if (signatureBuffer.length !== 64) { + throw new Error("INVALID_SIGNATURE_LENGTH"); + } + + if (Nacl.sign.detached.verify(signedBuffer, signatureBuffer, pubBuffer) !== true) { + throw new Error("FAILED"); + } +}; + +COMMANDS.DETACHED = function (data, cb) { + try { + checkDetachedSignature(data.msg, data.sig, data.key); + } catch (err) { + return void cb(err && err.message); + } + cb(); +}; + +COMMANDS.HASH_CHANNEL_LIST = function (data, cb) { + var channels = data.channels; + if (!Array.isArray(channels)) { return void cb('INVALID_CHANNEL_LIST'); } + var uniques = []; + + channels.forEach(function (a) { + if (uniques.indexOf(a) === -1) { uniques.push(a); } + }); + uniques.sort(); + + var hash = Nacl.util.encodeBase64(Nacl.hash(Nacl + .util.decodeUTF8(JSON.stringify(uniques)))); + + cb(void 0, hash); +}; + process.on('message', function (data) { if (!data || !data.txid || !data.pid) { return void process.send({ diff --git a/lib/workers/index.js b/lib/workers/index.js index d2944225c..02b3fb868 100644 --- a/lib/workers/index.js +++ b/lib/workers/index.js @@ -3,103 +3,14 @@ const Util = require("../common-util"); const nThen = require('nthen'); const OS = require("os"); -const numCPUs = OS.cpus().length; const { fork } = require('child_process'); const Workers = module.exports; const PID = process.pid; -const CRYPTO_PATH = 'lib/workers/crypto-worker'; const DB_PATH = 'lib/workers/db-worker'; +const MAX_JOBS = 16; -Workers.initializeValidationWorkers = function (Env) { - if (typeof(Env.validateMessage) !== 'undefined') { - return void console.error("validation workers are already initialized"); - } - - // Create our workers - const workers = []; - for (let i = 0; i < numCPUs; i++) { - workers.push(fork(CRYPTO_PATH)); - } - - const response = Util.response(function (errLabel, info) { - Env.Log.error('HK_VALIDATE_WORKER__' + errLabel, info); - }); - - var initWorker = function (worker) { - worker.on('message', function (res) { - if (!res || !res.txid) { return; } - response.handle(res.txid, [res.error, res.value]); - }); - - var substituteWorker = Util.once( function () { - Env.Log.info("SUBSTITUTE_VALIDATION_WORKER", ''); - var idx = workers.indexOf(worker); - if (idx !== -1) { - workers.splice(idx, 1); - } - // Spawn a new one - var w = fork(CRYPTO_PATH); - workers.push(w); - initWorker(w); - }); - - // Spawn a new process in one ends - worker.on('exit', substituteWorker); - worker.on('close', substituteWorker); - worker.on('error', function (err) { - substituteWorker(); - Env.Log.error('VALIDATION_WORKER_ERROR', { - error: err, - }); - }); - }; - workers.forEach(initWorker); - - var nextWorker = 0; - const send = function (msg, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - // let's be paranoid about asynchrony and only calling back once.. - nextWorker = (nextWorker + 1) % workers.length; - if (workers.length === 0 || typeof(workers[nextWorker].send) !== 'function') { - return void cb("INVALID_WORKERS"); - } - - var txid = msg.txid = Util.uid(); - - // expect a response within 45s - response.expect(txid, cb, 60000); - - // Send the request - workers[nextWorker].send(msg); - }; - - Env.validateMessage = function (signedMsg, key, cb) { - send({ - msg: signedMsg, - key: key, - command: 'INLINE', - }, cb); - }; - - Env.checkSignature = function (signedMsg, signature, publicKey, cb) { - send({ - command: 'DETACHED', - sig: signature, - msg: signedMsg, - key: publicKey, - }, cb); - }; - - Env.hashChannelList = function (channels, cb) { - send({ - command: 'HASH_CHANNEL_LIST', - channels: channels, - }, cb); - }; -}; - -Workers.initializeIndexWorkers = function (Env, config, _cb) { +Workers.initialize = function (Env, config, _cb) { var cb = Util.once(Util.mkAsync(_cb)); const workers = []; @@ -124,12 +35,18 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { return response.expected(id)? guid(): id; }; - const MAX_JOBS = 32; var workerOffset = -1; + var queue = []; var getAvailableWorkerIndex = function () { +// If there is already a backlog of tasks you can avoid some work +// by going to the end of the line + if (queue.length) { return -1; } + var L = workers.length; if (L === 0) { - console.log("no workers available"); // XXX + Log.error('NO_WORKERS_AVAILABLE', { + queue: queue.length, + }); return -1; } @@ -157,8 +74,6 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { return -1; }; - var queue = []; - var sendCommand = function (msg, _cb) { var index = getAvailableWorkerIndex(); @@ -383,11 +298,33 @@ Workers.initializeIndexWorkers = function (Env, config, _cb) { }, cb); }; + // Synchronous crypto functions + Env.validateMessage = function (signedMsg, key, cb) { + sendCommand({ + msg: signedMsg, + key: key, + command: 'INLINE', + }, cb); + }; + + Env.checkSignature = function (signedMsg, signature, publicKey, cb) { + sendCommand({ + command: 'DETACHED', + sig: signature, + msg: signedMsg, + key: publicKey, + }, cb); + }; + + Env.hashChannelList = function (channels, cb) { + sendCommand({ + command: 'HASH_CHANNEL_LIST', + channels: channels, + }, cb); + }; + cb(void 0); }); }; -Workers.initialize = function (Env, config, cb) { - Workers.initializeValidationWorkers(Env); - Workers.initializeIndexWorkers(Env, config, cb); -}; + From 0465f31a45dd1dca1cde9788b0add1d1b1d59460 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 16 Apr 2020 13:02:27 -0400 Subject: [PATCH 21/38] add a FIXME note --- lib/hk-util.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/hk-util.js b/lib/hk-util.js index b28cad497..e9943f1c1 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -672,6 +672,7 @@ const handleGetHistoryRange = function (Env, Server, seq, userId, parsed) { if (!Array.isArray(messages)) { messages = []; } + // FIXME this reduction could be done in the worker instead of the main process var toSend = []; if (typeof (desiredMessages) === "number") { toSend = messages.slice(-desiredMessages); From ba6faca02e7a3a8b953e1377f810da6e1a302c14 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 16 Apr 2020 13:53:45 -0400 Subject: [PATCH 22/38] make the number of workers configurable --- config/config.example.js | 8 ++++++++ lib/historyKeeper.js | 2 ++ lib/workers/index.js | 27 ++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/config/config.example.js b/config/config.example.js index 273c196d2..3826a7291 100644 --- a/config/config.example.js +++ b/config/config.example.js @@ -89,6 +89,14 @@ module.exports = { */ //httpSafePort: 3001, +/* CryptPad will launch a child process for every core available + * in order to perform CPU-intensive tasks in parallel. + * Some host environments may have a very large number of cores available + * or you may want to limit how much computing power CryptPad can take. + * If so, set 'maxWorkers' to a positive integer. + */ + // maxWorkers: 4, + /* ===================== * Admin * ===================== */ diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 46efcc436..85bc73ab1 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -253,6 +253,8 @@ module.exports.create = function (config, cb) { channelExpirationMs: config.channelExpirationMs, verbose: config.verbose, openFileLimit: config.openFileLimit, + + maxWorkers: config.maxWorkers, }, w(function (err) { if (err) { throw new Error(err); diff --git a/lib/workers/index.js b/lib/workers/index.js index 02b3fb868..2ad7377a4 100644 --- a/lib/workers/index.js +++ b/lib/workers/index.js @@ -193,7 +193,32 @@ Workers.initialize = function (Env, config, _cb) { }; nThen(function (w) { - OS.cpus().forEach(function () { + const max = config.maxWorkers; + + var limit; + if (typeof(max) !== 'undefined') { + // the admin provided a limit on the number of workers + if (typeof(max) === 'number' && !isNaN(max)) { + if (max < 1) { + Log.info("INSUFFICIENT_MAX_WORKERS", max); + limit = 1; + } + } else { + Log.error("INVALID_MAX_WORKERS", '[' + max + ']'); + } + } + + var logged; + + OS.cpus().forEach(function (cpu, index) { + if (limit && index >= limit) { + if (!logged) { + logged = true; + Log.info('WORKER_LIMIT', "(Opting not to use available CPUs beyond " + index + ')'); + } + return; + } + initWorker(fork(DB_PATH), w(function (err) { if (!err) { return; } w.abort(); From c39adb9bff586f1ac9c88779ae4ea60316532ffe Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 16 Apr 2020 14:59:40 -0400 Subject: [PATCH 23/38] configure ckeditor to allow unsetting font size --- customize.dist/ckeditor-config.js | 1 + 1 file changed, 1 insertion(+) diff --git a/customize.dist/ckeditor-config.js b/customize.dist/ckeditor-config.js index 5c4940c52..efbb116a4 100644 --- a/customize.dist/ckeditor-config.js +++ b/customize.dist/ckeditor-config.js @@ -11,6 +11,7 @@ CKEDITOR.editorConfig = function( config ) { config.removePlugins= 'resize,elementspath'; config.resize_enabled= false; //bottom-bar config.extraPlugins= 'autolink,colorbutton,colordialog,font,indentblock,justify,mediatag,print,blockbase64,mathjax,wordcount'; + config.fontSize_sizes = '(Default)/unset;8/8px;9/9px;10/10px;11/11px;12/12px;14/14px;16/16px;18/18px;20/20px;22/22px;24/24px;26/26px;28/28px;36/36px;48/48px;72/72px'; // XXX translation for default? config.toolbarGroups= [ // {"name":"clipboard","groups":["clipboard","undo"]}, //{"name":"editing","groups":["find","selection"]}, From 4e57e390da009ef266a25ea81534989af93e3b58 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 16 Apr 2020 16:37:24 -0400 Subject: [PATCH 24/38] write tasks inside of workers --- lib/historyKeeper.js | 6 ------ lib/hk-util.js | 2 +- lib/workers/db-worker.js | 5 +++++ lib/workers/index.js | 9 +++++++++ 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 85bc73ab1..ff0327751 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -31,7 +31,6 @@ module.exports.create = function (config, cb) { // and more easily share state between historyKeeper and rpc const Env = { Log: Log, - // tasks // store id: Crypto.randomBytes(8).toString('hex'), @@ -261,11 +260,6 @@ module.exports.create = function (config, cb) { } })); }).nThen(function (w) { - // create a task store (for scheduling tasks) - require("./storage/tasks").create(config, w(function (e, tasks) { - if (e) { throw e; } - Env.tasks = tasks; - })); if (config.disableIntegratedTasks) { return; } config.intervals = config.intervals || {}; diff --git a/lib/hk-util.js b/lib/hk-util.js index e9943f1c1..0e79c0691 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -529,7 +529,7 @@ const handleFirstMessage = function (Env, channelName, metadata) { if(metadata.expire && typeof(metadata.expire) === 'number') { // the fun part... // the user has said they want this pad to expire at some point - Env.tasks.write(metadata.expire, "EXPIRE", [ channelName ], function (err) { + Env.writeTask(metadata.expire, "EXPIRE", [ channelName ], function (err) { if (err) { // if there is an error, we don't want to crash the whole server... // just log it, and if there's a problem you'll be able to fix it diff --git a/lib/workers/db-worker.js b/lib/workers/db-worker.js index d76ce61dd..0f3d3b87f 100644 --- a/lib/workers/db-worker.js +++ b/lib/workers/db-worker.js @@ -419,6 +419,10 @@ const runTasks = function (data, cb) { Env.tasks.runAll(cb); }; +const writeTask = function (data, cb) { + Env.tasks.write(data.time, data.task_command, data.args, cb); +}; + const COMMANDS = { COMPUTE_INDEX: computeIndex, COMPUTE_METADATA: computeMetadata, @@ -431,6 +435,7 @@ const COMMANDS = { GET_HASH_OFFSET: getHashOffset, REMOVE_OWNED_BLOB: removeOwnedBlob, RUN_TASKS: runTasks, + WRITE_TASK: writeTask, }; COMMANDS.INLINE = function (data, cb) { diff --git a/lib/workers/index.js b/lib/workers/index.js index 2ad7377a4..cc11d538b 100644 --- a/lib/workers/index.js +++ b/lib/workers/index.js @@ -323,6 +323,15 @@ Workers.initialize = function (Env, config, _cb) { }, cb); }; + Env.writeTask = function (time, command, args, cb) { + sendCommand({ + command: 'WRITE_TASK', + time: time, + task_command: command, + args: args, + }, cb); + }; + // Synchronous crypto functions Env.validateMessage = function (signedMsg, key, cb) { sendCommand({ From 78afbaa6a88a0ca207f418eefd33a2179f71c41b Mon Sep 17 00:00:00 2001 From: Weblate Date: Fri, 17 Apr 2020 17:06:07 +0200 Subject: [PATCH 25/38] Translated using Weblate (English) Currently translated at 100.0% (1254 of 1254 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/en/ Translated using Weblate (English) Currently translated at 100.0% (1253 of 1253 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/en/ Translated using Weblate (English) Currently translated at 100.0% (1252 of 1252 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/en/ Translated using Weblate (English) Currently translated at 100.0% (1252 of 1252 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/en/ Translated using Weblate (English) Currently translated at 100.0% (1251 of 1251 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/en/ Translated using Weblate (English) Currently translated at 100.0% (1250 of 1250 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/en/ Translated using Weblate (English) Currently translated at 100.0% (1249 of 1249 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/en/ Translated using Weblate (English) Currently translated at 100.0% (1249 of 1249 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/en/ Translated using Weblate (English) Currently translated at 100.0% (1248 of 1248 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/en/ Translated using Weblate (English) Currently translated at 100.0% (1247 of 1247 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/en/ --- www/common/translations/messages.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/www/common/translations/messages.json b/www/common/translations/messages.json index 50133d1a2..19c85eb3d 100644 --- a/www/common/translations/messages.json +++ b/www/common/translations/messages.json @@ -1342,5 +1342,13 @@ "admin_openFilesTitle": "Open Files", "admin_openFilesHint": "Number of file descriptors currently open on the server.", "canvas_brush": "Brush", - "canvas_select": "Select" + "canvas_select": "Select", + "cba_writtenBy": "Written by: {0}", + "cba_properties": "Author colors (experimental)", + "cba_hint": "This setting will be remembered when you create your next pad.", + "cba_enable": "Enable", + "cba_disable": "Clear and Disable", + "cba_show": "Show author colors", + "cba_hide": "Hide author colors", + "oo_login": "Please log in or register to improve the performance of spreadsheets." } From 8b0006fba74b830d6caf71a4e7dcd509c4cc1ab7 Mon Sep 17 00:00:00 2001 From: Weblate Date: Fri, 17 Apr 2020 17:06:07 +0200 Subject: [PATCH 26/38] Translated using Weblate (French) Currently translated at 99.4% (1246 of 1254 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/fr/ --- www/common/translations/messages.fr.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/www/common/translations/messages.fr.json b/www/common/translations/messages.fr.json index 17582944a..b1ee72a49 100644 --- a/www/common/translations/messages.fr.json +++ b/www/common/translations/messages.fr.json @@ -1342,5 +1342,12 @@ "admin_openFilesTitle": "Fichiers Ouverts", "profile_copyKey": "Copier la clé publique", "canvas_select": "Selection", - "canvas_brush": "Pinceau" + "canvas_brush": "Pinceau", + "cba_show": "", + "cba_disable": "", + "cba_hint": "", + "cba_enable": "", + "cba_writtenBy": "", + "cba_properties": "", + "cba_hide": "" } From 5807f3680b0b480b9ad4ca621192f188f40a7d6c Mon Sep 17 00:00:00 2001 From: Weblate Date: Fri, 17 Apr 2020 17:06:07 +0200 Subject: [PATCH 27/38] Translated using Weblate (German) Currently translated at 99.4% (1246 of 1254 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/de/ --- www/common/translations/messages.de.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/www/common/translations/messages.de.json b/www/common/translations/messages.de.json index 86e2300e4..b100a141c 100644 --- a/www/common/translations/messages.de.json +++ b/www/common/translations/messages.de.json @@ -1342,5 +1342,11 @@ "admin_openFilesTitle": "Offene Dateien", "canvas_select": "Auswahl", "canvas_brush": "Pinsel", - "profile_copyKey": "Öffentlichen Schlüssel kopieren" + "profile_copyKey": "Öffentlichen Schlüssel kopieren", + "cba_hide": "", + "cba_enable": "", + "cba_properties": "", + "cba_disable": "", + "cba_show": "", + "cba_writtenBy": "" } From fab1fcdb3306c26c2c4a4cb20b08f9b7cb115df3 Mon Sep 17 00:00:00 2001 From: Weblate Date: Fri, 17 Apr 2020 17:06:07 +0200 Subject: [PATCH 28/38] Translated using Weblate (Dutch) Currently translated at 30.2% (376 of 1246 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/nl/ --- www/common/translations/messages.nl.json | 178 ++++++++++++++++++++++- 1 file changed, 176 insertions(+), 2 deletions(-) diff --git a/www/common/translations/messages.nl.json b/www/common/translations/messages.nl.json index 7c18658d9..e2da9e201 100644 --- a/www/common/translations/messages.nl.json +++ b/www/common/translations/messages.nl.json @@ -201,6 +201,180 @@ "chainpadError": "Er was een kritieke fout bij het updaten van uw inhoud. Deze pagina is nu alleen leesbaar om uw werk niet kwijt te raken.
Druk op Esc om verder te gaan met het bekijken van deze werkomgeving, of herlaad de pagina om te proberen deze werkomgeving weer aan te kunnen passen.", "inactiveError": "Deze werkomgeving is verwijderd wegens gebrek aan activiteit. Druk op Esc om een nieuwe werkomgeving aan te maken.", "deletedError": "Deze werkomgeving is verwijderd door de eigenaar en is niet meer beschikbaar.", - "main_title": "CryptPad: Geen Kennis, Onvertraagd Collaboratief Aanpassen" + "main_title": "CryptPad: Geen Kennis, Onvertraagd Collaboratief Aanpassen", + "fc_open_ro": "Openen (alleen-lezen)", + "history_restorePrompt": "Weet u zeker dat u de bestaande documentversie wilt vervangen met de weergegeven versie?", + "history_restoreDone": "Document hersteld", + "history_version": "Versie:", + "openLinkInNewTab": "Open Link in Nieuw Tabblad", + "pad_mediatagTitle": "Mediamarkering Instellingen", + "pad_mediatagWidth": "Breedte (px)", + "pad_mediatagHeight": "Hoogte (px)", + "pad_mediatagRatio": "Behoud ratio", + "pad_mediatagBorder": "Randbreedte (px)", + "pad_mediatagPreview": "Voorbeeld", + "pad_mediatagImport": "Opslaan in uw CryptDrive", + "pad_mediatagOptions": "Afbeeldingeigenschappen", + "kanban_newBoard": "Nieuw bord", + "kanban_item": "Item {0}", + "kanban_todo": "Te Doen", + "kanban_working": "In voortgang", + "kanban_deleteBoard": "Weet u zeker dat u dit bord wilt verwijderen?", + "kanban_addBoard": "Een bord toevoegen", + "kanban_removeItem": "Verwijder dit item", + "kanban_removeItemConfirm": "Weet u zeker dat u dit item wilt verwijderen?", + "poll_p_save": "Uw instellingen worden direct geüpdate, dus u hoeft deze nooit op te slaan.", + "poll_p_encryption": "Al uw invoer worden versleuteld zodat alleen de personen met de link toegang kunnen krijgen. Zelfs de server kan niet zien wat u wijzigt.", + "wizardLog": "Klik op de knop linksboven om terug te gaan naar uw enquête", + "wizardTitle": "Gebruik het formulier om een enquête aan te maken", + "wizardConfirm": "Bent u echt klaar om deze opties toe te voegen aan uw enquête?", + "poll_publish_button": "Publiceren", + "poll_admin_button": "Admin", + "poll_create_user": "Voeg een nieuwe gebruiker toe", + "poll_create_option": "Voeg een nieuwe optie toe", + "poll_commit": "Indienen", + "poll_closeWizardButton": "Sluit formulier", + "poll_closeWizardButtonTitle": "Sluit formulier", + "poll_wizardComputeButton": "Bereken Opties", + "poll_wizardClearButton": "Tabel Leegmaken", + "poll_wizardAddDateButton": "+ Data", + "poll_wizardAddTimeButton": "+ Tijden", + "poll_optionPlaceholder": "Optie", + "poll_userPlaceholder": "Uw naam", + "poll_removeOption": "Weet u zeker dat u deze optie wilt verwijderen?", + "poll_removeUser": "Weet u zeker dat u deze gebruiker wilt verwijderen?", + "poll_titleHint": "Titel", + "poll_remove": "Verwijder", + "poll_edit": "Wijzig", + "poll_locked": "Vergrendeld", + "poll_unlocked": "Ontgrendeld", + "poll_bookmark_col": "Voeg een bladwijzer toe aan deze kolom zodat deze altijd ontgrendeld is en weergegeven aan het begin", + "poll_bookmarked_col": "Dit is uw kolom met bladwijzer. Deze is altijd ontgrendeld en weergegeven aan het begin.", + "poll_total": "TOTAAL", + "poll_comment_list": "Commentaar", + "poll_comment_add": "Commentaar toevoegen", + "poll_comment_submit": "Verzenden", + "poll_comment_placeholder": "Uw commentaar", + "poll_comment_disabled": "Publiceer deze enquête met de ✓ knop om commentaar toe te staan.", + "oo_reconnect": "De verbinding met de server is terug. Klik op OK om te herladen en door te gaan met werken.", + "oo_cantUpload": "Uploaden is niet toegestaan als andere gebruikers aanwezig zijn.", + "oo_uploaded": "Uw upload is voltooid. Klik op OK om te herladen of Annuleren om door te gaan in alleen-lezen modus.", + "canvas_clear": "Leegmaken", + "canvas_delete": "Verwijder selectie", + "canvas_disable": "Tekenen uitschakelen", + "canvas_enable": "Tekenen inschakelen", + "canvas_width": "Breedte", + "canvas_opacity": "Ondoorzichtigheid", + "canvas_opacityLabel": "Ondoorzichtigheid: {0}", + "canvas_widthLabel": "Breedte: {0}", + "canvas_saveToDrive": "Deze afbeelding opslaan als bestand in uw CryptDrive", + "canvas_currentBrush": "Huidige borstel", + "canvas_chooseColor": "Kies een kleur", + "canvas_imageEmbed": "Plaats een afbeelding van uw computer", + "profileButton": "Profiel", + "profile_urlPlaceholder": "URL", + "profile_namePlaceholder": "Naam weergegeven op uw profiel", + "profile_avatar": "Avatar", + "profile_upload": " Een nieuw avatar uploaden", + "profile_uploadSizeError": "Foutmelding: uw avatar moet kleiner zijn dan {0}", + "profile_uploadTypeError": "Foutmelding: uw avatartype is niet toegestaan. Toegestane types zijn: {0}", + "profile_error": "Foutmelding tijdens het maken van uw profiel: {0}", + "profile_register": "U moet zich aanmelden om een profiel aan te maken!", + "profile_create": "Een profiel aanmaken", + "profile_description": "Beschrijving", + "profile_viewMyProfile": "Mijn profiel bekijken", + "userlist_addAsFriendTitle": "Stuur \"{0}\" een contactverzoek", + "contacts_title": "Contacten", + "contacts_addError": "Foutmelding tijdens het toevoegen van die contact aan de lijst", + "contacts_added": "Contactuitnodiging geaccepteerd.", + "contacts_rejected": "Contactuitnodiging afgewezen", + "contacts_request": "{0} wilt u toevoegen als contactpersoon. Accepteren?", + "contacts_send": "Verzenden", + "contacts_remove": "Verwijder deze contactpersoon", + "contacts_confirmRemove": "Weet u zeker dat u {0} wilt verwijderen uit uw contacten?", + "contacts_typeHere": "Schrijf hier een bericht...", + "contacts_warning": "Alles dat u hier schrijft blijft beschikbaar voor alle huidige en toekomstige gebruikers van deze werkomgeving. Wees voorzichtig met gevoelige informatie!", + "contacts_padTitle": "Gesprek", + "contacts_info1": "Dit zijn uw contacten. Vanaf hier kunt u:", + "contacts_info2": "Klik op uw contactpersoon's icoon om een gesprek met hun te beginnen", + "contacts_info3": "Dubbelklik op hun icoon om hun profiel te bekijken", + "contacts_info4": "Beide deelnemers kunnen de gesprekgeschiedenis voorgoed leegmaken", + "contacts_removeHistoryTitle": "Gesprekgeschiedenis leegmaken", + "contacts_confirmRemoveHistory": "Weet u zeker dat u voorgoed de gesprekgeschiedenis wilt leegmaken? De data kan niet worden hersteld", + "contacts_removeHistoryServerError": "Foutmelding tijdens het verwijderen van uw gesprekgeschiedenis. Probeer later opnieuw", + "contacts_fetchHistory": "Eerdere geschiedenis binnenhalen", + "contacts_friends": "Contacten", + "contacts_rooms": "Kamers", + "contacts_leaveRoom": "Deze kamer verlaten", + "contacts_online": "Een andere gebruiker in deze kamer is online", + "debug_getGraphWait": "De grafiek genereren... Even wachten, alstublieft.", + "fm_rootName": "Documenten", + "fm_trashName": "Prullenbak", + "fm_unsortedName": "Niet-gesorteerde bestanden", + "fm_filesDataName": "Alle bestanden", + "fm_templateName": "Sjablonen", + "fm_searchName": "Zoeken", + "fm_recentPadsName": "Recente werkomgevingen", + "fm_tagsName": "Markeringen", + "fm_sharedFolderName": "Gedeelde map", + "fm_searchPlaceholder": "Zoeken...", + "fm_newButton": "Nieuw", + "fm_newButtonTitle": "Make een nieuwe werkomgeving of map, importeer een bestand in de huidige map", + "fm_newFolder": "Nieuwe map", + "fm_newFile": "Nieuwe werkomgeving", + "fm_folder": "Map", + "fm_sharedFolder": "Gedeelde map", + "fm_folderName": "Mapnaam", + "fm_numberOfFolders": "Aantal mappen", + "fm_numberOfFiles": "Aantal bestanden", + "fm_fileName": "Bestandsnaam", + "fm_title": "Titel", + "fm_type": "Type", + "fm_forbidden": "Verboden actie", + "fm_originalPath": "Oorspronkelijke pad", + "fm_openParent": "Weergeven in map", + "fm_noname": "Document Zonder Title", + "fm_emptyTrashDialog": "Weet u zeker dat u de prullenbak wilt legen?", + "fm_removeSeveralPermanentlyDialog": "Weet u zeker dat u deze {0} elementen voorgoed wilt verwijderen van uw CryptDrive?", + "fm_removePermanentlyDialog": "Weet u zeker dat u dit element voorgoed wilt verwijderen van uw CryptDrive?", + "fm_removePermanentlyNote": "Werkomgevingen in uw eigendom worden verwijderd van de server als u doorgaat.", + "fm_removeSeveralDialog": "Weet u zeker dat u deze {0} elementen wilt verplaatsen naar de prullenbak?", + "fm_removeDialog": "Weet u zeker dat u {0} wilt verplaatsen naar de prullenbak?", + "fm_deleteOwnedPad": "Weet u zeker dat u deze werkomgeving wilt verwijderen van de server?", + "fm_deleteOwnedPads": "Weet u zeker dat u deze werkomgevingen wilt verwijderen van de server?", + "fm_restoreDialog": "Weet u zeker dat u {0} wilt terugzetten naar de vorige lokatie?", + "fm_unknownFolderError": "De geselecteerde of laatst bezochte map bestaat niet meer. De bovenliggende map aan het openen...", + "fm_backup_title": "Backup link", + "fm_nameFile": "Hoe wilt u dit bestand noemen?", + "fm_error_cantPin": "Interne server fout. Graag de pagina herladen en opnieuw proberen.", + "fm_canBeShared": "Deze map is deelbaar", + "fm_prop_tagsList": "Markeringen", + "fm_burnThisDriveButton": "Verwijder alle informatie opgeslagen door CryptPad in uw browser", + "fm_padIsOwned": "U bent de eigenaar van deze werkomgeving", + "fm_padIsOwnedOther": "Deze werkomgeving is van een andere gebruiker", + "fm_deletedPads": "Deze werkomgeving bestaan niet meer op de server, ze zijn verwijderd van uw CryptDrive: {0}", + "fm_tags_name": "Markeringsnaam", + "fm_tags_used": "Aantal gebruikers", + "fm_passwordProtected": "Vergrendeld met wachtwoord", + "fc_newfolder": "Nieuwe map", + "fc_newsharedfolder": "Nieuwe gedeelde map", + "fc_rename": "Hernoemen", + "fc_color": "Kleur wijzigen", + "fc_open": "Openen", + "fc_openInCode": "Openen in Code bewerker", + "fc_expandAll": "Alles Uitvouwen", + "fc_collapseAll": "Alles Samenvouwen", + "fc_delete": "Naar prullenbak verplaatsen", + "fc_delete_owned": "Verwijderen van server", + "fc_restore": "Herstellen", + "fc_remove": "Verwijder uit uw CryptDrive", + "fc_remove_sharedfolder": "Verwijderen", + "fc_empty": "Prullenbak legen", + "fc_prop": "Eigenschappen", + "fc_hashtag": "Markeringen", + "getEmbedCode": "Krijg integratiecode", + "kanban_done": "Gedaan", + "poll_comment_remove": "Verwijder dit commentaar", + "profile_fieldSaved": "Nieuwe waarde opgeslagen: {0}", + "fm_morePads": "Meer" } - From 8009c01f704bf1fcc4d9ec05ee2322972dc468bb Mon Sep 17 00:00:00 2001 From: Weblate Date: Fri, 17 Apr 2020 17:22:13 +0200 Subject: [PATCH 29/38] Translated using Weblate (French) Currently translated at 100.0% (1254 of 1254 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/fr/ --- www/common/translations/messages.fr.json | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/www/common/translations/messages.fr.json b/www/common/translations/messages.fr.json index b1ee72a49..083ce8ad6 100644 --- a/www/common/translations/messages.fr.json +++ b/www/common/translations/messages.fr.json @@ -1343,11 +1343,12 @@ "profile_copyKey": "Copier la clé publique", "canvas_select": "Selection", "canvas_brush": "Pinceau", - "cba_show": "", - "cba_disable": "", - "cba_hint": "", - "cba_enable": "", - "cba_writtenBy": "", - "cba_properties": "", - "cba_hide": "" + "cba_show": "Montrer les couleurs d'auteurs", + "cba_disable": "Effacer et Désactiver", + "cba_hint": "Ce réglage sera mémorisé lors de la création de votre prochain pad.", + "cba_enable": "Activer", + "cba_writtenBy": "Écrit par : {0}", + "cba_properties": "Couleurs par auteurs (expérimental)", + "cba_hide": "Cacher les couleurs d'auteurs", + "oo_login": "Veuillez vous connecter ou vous inscrire pour améliorer la performance des feuilles de calcul." } From d0c3f0fa84daed198dbeb1f5d99afc75fe35de9b Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 17 Apr 2020 11:36:48 -0400 Subject: [PATCH 30/38] remove XXX notes --- www/common/onlyoffice/inner.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/www/common/onlyoffice/inner.js b/www/common/onlyoffice/inner.js index 20125d82c..dd34585d8 100644 --- a/www/common/onlyoffice/inner.js +++ b/www/common/onlyoffice/inner.js @@ -355,7 +355,6 @@ define([ APP.FM.handleFile(blob, data); }; - Messages.oo_login = 'Log in...'; // XXX var noLogin = false; var makeCheckpoint = function (force) { @@ -1120,12 +1119,12 @@ define([ var x2tSaveAndConvertData = function(data, filename, extension, finalFilename) { // Perform the x2t conversion - require(['/common/onlyoffice/x2t/x2t.js'], function() { // XXX why does this fail without an access-control-allow-origin header? + require(['/common/onlyoffice/x2t/x2t.js'], function() { // FIXME why does this fail without an access-control-allow-origin header? var x2t = window.Module; x2t.run(); if (x2tInitialized) { debug("x2t runtime already initialized"); - x2tSaveAndConvertDataInternal(x2t, data, filename, extension, finalFilename); // XXX shouldn't this return ? + return void x2tSaveAndConvertDataInternal(x2t, data, filename, extension, finalFilename); } x2t.onRuntimeInitialized = function() { From adae4b690b471e9b4b233d9cdec9d3e59bda175c Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 17 Apr 2020 17:38:15 +0200 Subject: [PATCH 31/38] outer.css cache busting --- www/admin/index.html | 2 +- www/code/index.html | 2 +- www/contacts/index.html | 2 +- www/debug/index.html | 2 +- www/drive/index.html | 2 +- www/file/index.html | 2 +- www/kanban/index.html | 2 +- www/notifications/index.html | 2 +- www/oodoc/index.html | 2 +- www/ooslide/index.html | 2 +- www/pad/index.html | 2 +- www/poll/index.html | 2 +- www/profile/index.html | 2 +- www/settings/index.html | 2 +- www/sheet/index.html | 2 +- www/slide/index.html | 2 +- www/support/index.html | 2 +- www/teams/index.html | 2 +- www/todo/index.html | 2 +- www/whiteboard/index.html | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/www/admin/index.html b/www/admin/index.html index 79a96c97b..c59dd8edc 100644 --- a/www/admin/index.html +++ b/www/admin/index.html @@ -6,7 +6,7 @@ - +