From 5c53868c3bbd6bca0541c1ea44ff767af56c6da1 Mon Sep 17 00:00:00 2001 From: yflory Date: Mon, 26 Feb 2018 18:23:12 +0100 Subject: [PATCH] Delete pads after 3 months of inactivity --- config.example.js | 8 + .../src/less2/include/colortheme.less | 6 +- customize.dist/src/less2/include/toolbar.less | 2 + customize.dist/translations/messages.fr.js | 3 +- customize.dist/translations/messages.js | 3 +- delete-inactive.js | 40 ++++ expire-channels.js | 1 - pinneddata.js | 205 ++++++++++-------- www/common/common-interface.js | 8 +- www/common/sframe-app-framework.js | 10 +- www/common/sframe-common-outer.js | 7 +- www/common/sframe-common.js | 24 ++ www/common/toolbar3.js | 2 +- www/poll/inner.js | 11 +- www/whiteboard/inner.js | 10 +- 15 files changed, 211 insertions(+), 129 deletions(-) create mode 100644 delete-inactive.js diff --git a/config.example.js b/config.example.js index 7e728451a..3aace0d4a 100644 --- a/config.example.js +++ b/config.example.js @@ -249,6 +249,14 @@ module.exports = { */ pinPath: './pins', + /* Pads that are not 'pinned' by any registered user can be set to expire + * after a configurable number of days of inactivity (default 90 days). + * The value can be changed or set to false to remove expiration. + * Expired pads can then be removed using a cron job calling the + * `delete-inactive.js` script with node + */ + inactiveTime: 90, // days + /* CryptPad allows logged in users to upload encrypted files. Files/blobs * are stored in a 'blob-store'. Set its location here. */ diff --git a/customize.dist/src/less2/include/colortheme.less b/customize.dist/src/less2/include/colortheme.less index c8848d02b..1a2b143e5 100644 --- a/customize.dist/src/less2/include/colortheme.less +++ b/customize.dist/src/less2/include/colortheme.less @@ -44,11 +44,11 @@ @colortheme_pad-bg: #1c4fa0; @colortheme_pad-color: #fff; @colortheme_pad-toolbar-bg: #c1e7ff; -@colortheme_pad-warn: #F83A3A; +@colortheme_pad-warn: #ffae00; @colortheme_slide-bg: #e57614; @colortheme_slide-color: #fff; -@colortheme_slide-warn: #58D697; +@colortheme_slide-warn: #005868; @colortheme_code-bg: #ffae00; @colortheme_code-color: #000; @@ -59,7 +59,7 @@ @colortheme_poll-help-bg: #bbffbb; @colortheme_poll-th-bg: #005bef; @colortheme_poll-th-fg: #fff; -@colortheme_poll-warn: #ffae00; +@colortheme_poll-warn: #ffade3; @colortheme_whiteboard-bg: #800080; @colortheme_whiteboard-color: #fff; diff --git a/customize.dist/src/less2/include/toolbar.less b/customize.dist/src/less2/include/toolbar.less index 435476090..f8cd43fd0 100644 --- a/customize.dist/src/less2/include/toolbar.less +++ b/customize.dist/src/less2/include/toolbar.less @@ -505,6 +505,7 @@ font-size: @colortheme_app-font-size; a { font-size: @colortheme_app-font-size; + font-family: @colortheme_font; font-weight: bold; color: @warn-color; &:hover { @@ -792,6 +793,7 @@ } .cp-toolbar-share-button { width: 50px; + text-align: center; } } .cp-toolbar-rightside { diff --git a/customize.dist/translations/messages.fr.js b/customize.dist/translations/messages.fr.js index 2d21cf9a6..5e681e699 100644 --- a/customize.dist/translations/messages.fr.js +++ b/customize.dist/translations/messages.fr.js @@ -29,11 +29,12 @@ define(function () { out.typeError = "Ce pad n'est pas compatible avec l'application sélectionnée"; out.onLogout = 'Vous êtes déconnecté de votre compte utilisateur, cliquez ici pour vous authentifier
ou appuyez sur Échap pour accéder au pad en mode lecture seule.'; out.wrongApp = "Impossible d'afficher le contenu de ce document temps-réel dans votre navigateur. Vous pouvez essayer de recharger la page."; - out.padNotPinned = 'Ce pad va expirer dans 3 mois, {0}connectez-vous{1} ou {2}enregistrez-vous{3} pour le préserver.'; + out.padNotPinned = 'Ce pad va expirer après 3 mois d\'inactivité, {0}connectez-vous{1} ou {2}enregistrez-vous{3} pour le préserver.'; out.anonymousStoreDisabled = "L'administrateur de cette instance de CryptPad a désactivé le drive pour les utilisateurs non enregistrés. Vous devez vous connecter pour pouvoir utiliser CryptDrive."; out.expiredError = "Ce pad a atteint sa date d'expiration est n'est donc plus disponible."; out.expiredErrorCopy = ' Vous pouvez toujours copier son contenu ailleurs en appuyant sur Échap.
Dés que vous aurez quitté la page, il sera impossible de le récupérer.'; out.deletedError = 'Ce pad a été supprimé par son propriétaire et n\'est donc plus disponible.'; + out.inactiveError = 'Ce pad a été supprimé en raison de son inactivité. Appuyez sur Échap pour créer un nouveau pad.'; out.loading = "Chargement..."; out.error = "Erreur"; diff --git a/customize.dist/translations/messages.js b/customize.dist/translations/messages.js index eaceee812..c5f7bcd8c 100644 --- a/customize.dist/translations/messages.js +++ b/customize.dist/translations/messages.js @@ -30,11 +30,12 @@ define(function () { out.typeError = "This pad is not compatible with the selected application"; out.onLogout = 'You are logged out, click here to log in
or press Escape to access your pad in read-only mode.'; out.wrongApp = "Unable to display the content of that realtime session in your browser. Please try to reload that page."; - out.padNotPinned = 'This pad will expire in 3 months, {0}login{1} or {2}register{3} to preserve it.'; + out.padNotPinned = 'This pad will expire after 3 months of inactivity, {0}login{1} or {2}register{3} to preserve it.'; out.anonymousStoreDisabled = "The webmaster of this CryptPad instance has disabled the store for anonymous users. You have to log in to be able to use CryptDrive."; out.expiredError = 'This pad has reached its expiration time and is no longer available.'; out.expiredErrorCopy = ' You can still copy the content to another location by pressing Esc.
Once you leave this page, it will disappear forever!'; out.deletedError = 'This pad has been deleted by its owner and is no longer available.'; + out.inactiveError = 'This pad has been deleted due to inactivity. Press Esc to create a new pad.'; out.loading = "Loading..."; out.error = "Error"; diff --git a/delete-inactive.js b/delete-inactive.js new file mode 100644 index 000000000..16f41da45 --- /dev/null +++ b/delete-inactive.js @@ -0,0 +1,40 @@ +/* jshint esversion: 6, node: true */ +const Fs = require("fs"); +const nThen = require("nthen"); +const Saferphore = require("saferphore"); +const PinnedData = require('./pinneddata'); +let config; +try { + config = require('./config'); +} catch (e) { + config = require('./config.example'); +} + +if (!config.inactiveTime || typeof(config.inactiveTime) !== "number") { return; } + +let inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000); +let inactiveConfig = { + unpinned: true, + olderthan: inactiveTime, + blobsolderthan: inactiveTime +}; +let toDelete; +nThen(function (waitFor) { + PinnedData.load(inactiveConfig, waitFor(function (err, data) { + if (err) { + waitFor.abort(); + throw new Error(err); + } + toDelete = data; + })); +}).nThen(function () { + var sem = Saferphore.create(10); + toDelete.forEach(function (f) { + sem.take(function (give) { + Fs.unlink(f.filename, give(function (err) { + if (err) { return void console.error(err + " " + f.filename); } + console.log(f.filename + " " + f.size + " " + (+f.atime) + " " + (+new Date())); + })); + }); + }); +}); diff --git a/expire-channels.js b/expire-channels.js index 4e22dfce4..a42a3af58 100644 --- a/expire-channels.js +++ b/expire-channels.js @@ -7,7 +7,6 @@ var config; try { config = require('./config'); } catch (e) { - console.log("You can customize the configuration by copying config.example.js to config.js"); config = require('./config.example'); } diff --git a/pinneddata.js b/pinneddata.js index d7848c7df..378f34ad7 100644 --- a/pinneddata.js +++ b/pinneddata.js @@ -47,103 +47,126 @@ const dsFileStats = {}; const out = []; const pinned = {}; -nThen((waitFor) => { - Fs.readdir('./datastore', waitFor((err, list) => { - if (err) { throw err; } - dirList = list; - })); -}).nThen((waitFor) => { - dirList.forEach((f) => { - sema.take((returnAfter) => { - Fs.readdir('./datastore/' + f, waitFor(returnAfter((err, list2) => { - if (err) { throw err; } - list2.forEach((ff) => { fileList.push('./datastore/' + f + '/' + ff); }); - }))); +module.exports.load = function (config, cb) { + nThen((waitFor) => { + Fs.readdir('./datastore', waitFor((err, list) => { + if (err) { throw err; } + dirList = list; + })); + }).nThen((waitFor) => { + dirList.forEach((f) => { + sema.take((returnAfter) => { + Fs.readdir('./datastore/' + f, waitFor(returnAfter((err, list2) => { + if (err) { throw err; } + list2.forEach((ff) => { fileList.push('./datastore/' + f + '/' + ff); }); + }))); + }); }); - }); -}).nThen((waitFor) => { + }).nThen((waitFor) => { - Fs.readdir('./blob', waitFor((err, list) => { - if (err) { throw err; } - dirList = list; - })); -}).nThen((waitFor) => { - dirList.forEach((f) => { - sema.take((returnAfter) => { - Fs.readdir('./blob/' + f, waitFor(returnAfter((err, list2) => { - if (err) { throw err; } - list2.forEach((ff) => { fileList.push('./blob/' + f + '/' + ff); }); - }))); + Fs.readdir('./blob', waitFor((err, list) => { + if (err) { throw err; } + dirList = list; + })); + }).nThen((waitFor) => { + dirList.forEach((f) => { + sema.take((returnAfter) => { + Fs.readdir('./blob/' + f, waitFor(returnAfter((err, list2) => { + if (err) { throw err; } + list2.forEach((ff) => { fileList.push('./blob/' + f + '/' + ff); }); + }))); + }); }); - }); -}).nThen((waitFor) => { - fileList.forEach((f) => { - sema.take((returnAfter) => { - Fs.stat(f, waitFor(returnAfter((err, st) => { - if (err) { throw err; } - st.filename = f; - dsFileStats[f.replace(/^.*\/([^\/\.]*)(\.ndjson)?$/, (all, a) => (a))] = st; - }))); + }).nThen((waitFor) => { + fileList.forEach((f) => { + sema.take((returnAfter) => { + Fs.stat(f, waitFor(returnAfter((err, st) => { + if (err) { throw err; } + st.filename = f; + dsFileStats[f.replace(/^.*\/([^\/\.]*)(\.ndjson)?$/, (all, a) => (a))] = st; + }))); + }); }); - }); -}).nThen((waitFor) => { - Fs.readdir('./pins', waitFor((err, list) => { - if (err) { throw err; } - dirList = list; - })); -}).nThen((waitFor) => { - fileList.splice(0, fileList.length); - dirList.forEach((f) => { - sema.take((returnAfter) => { - Fs.readdir('./pins/' + f, waitFor(returnAfter((err, list2) => { - if (err) { throw err; } - list2.forEach((ff) => { fileList.push('./pins/' + f + '/' + ff); }); - }))); + }).nThen((waitFor) => { + Fs.readdir('./pins', waitFor((err, list) => { + if (err) { throw err; } + dirList = list; + })); + }).nThen((waitFor) => { + fileList.splice(0, fileList.length); + dirList.forEach((f) => { + sema.take((returnAfter) => { + Fs.readdir('./pins/' + f, waitFor(returnAfter((err, list2) => { + if (err) { throw err; } + list2.forEach((ff) => { fileList.push('./pins/' + f + '/' + ff); }); + }))); + }); }); - }); -}).nThen((waitFor) => { - fileList.forEach((f) => { - sema.take((returnAfter) => { - Fs.readFile(f, waitFor(returnAfter((err, content) => { - if (err) { throw err; } - const hashes = hashesFromPinFile(content.toString('utf8'), f); - const size = sizeForHashes(hashes, dsFileStats); - if (process.argv.indexOf('--unpinned') > -1) { - hashes.forEach((x) => { pinned[x] = 1; }); - } else { - out.push([f, Math.floor(size / (1024 * 1024))]); - } - }))); + }).nThen((waitFor) => { + fileList.forEach((f) => { + sema.take((returnAfter) => { + Fs.readFile(f, waitFor(returnAfter((err, content) => { + if (err) { throw err; } + const hashes = hashesFromPinFile(content.toString('utf8'), f); + const size = sizeForHashes(hashes, dsFileStats); + if (config.unpinned) { + hashes.forEach((x) => { pinned[x] = 1; }); + } else { + out.push([f, Math.floor(size / (1024 * 1024))]); + } + }))); + }); }); - }); -}).nThen(() => { - if (process.argv.indexOf('--unpinned') > -1) { - const ot = process.argv.indexOf('--olderthan'); - let before = Infinity; - if (ot > -1) { - before = new Date(process.argv[ot+1]); - if (isNaN(before)) { - throw new Error('--olderthan error [' + process.argv[ot+1] + '] not a valid date'); + }).nThen(() => { + if (config.unpinned) { + let before = Infinity; + if (config.olderthan) { + before = config.olderthan; + if (isNaN(before)) { + return void cb('--olderthan error [' + config.olderthan + '] not a valid date'); + } } - } - const bot = process.argv.indexOf('--blobsolderthan'); - let blobsbefore = before; - if (bot > -1) { - blobsbefore = new Date(process.argv[bot+1]); - if (isNaN(blobsbefore)) { - throw new Error('--blobsolderthan error [' + process.argv[bot+1] + '] not a valid date'); + let blobsbefore = before; + if (config.blobsolderthan) { + blobsbefore = config.blobsolderthan; + if (isNaN(blobsbefore)) { + return void cb('--blobsolderthan error [' + config.blobsolderthan + '] not a valid date'); + } } + let files = []; + Object.keys(dsFileStats).forEach((f) => { + if (!(f in pinned)) { + const isBlob = dsFileStats[f].filename.indexOf('.ndjson') === -1; + if ((+dsFileStats[f].atime) >= ((isBlob) ? blobsbefore : before)) { return; } + files.push({ + filename: dsFileStats[f].filename, + size: dsFileStats[f].size, + atime: dsFileStats[f].atime + }); + } + }); + cb(null, files); + } else { + out.sort((a,b) => (a[1] - b[1])); + cb(null, out.slice()); } - Object.keys(dsFileStats).forEach((f) => { - if (!(f in pinned)) { - const isBlob = dsFileStats[f].filename.indexOf('.ndjson') === -1; - if ((+dsFileStats[f].mtime) >= ((isBlob) ? blobsbefore : before)) { return; } - console.log(dsFileStats[f].filename + " " + dsFileStats[f].size + " " + - (+dsFileStats[f].mtime)); - } - }); - } else { - out.sort((a,b) => (a[1] - b[1])); - out.forEach((x) => { console.log(x[0] + ' ' + x[1] + ' MB'); }); - } -}); + }); +}; + +if (!module.parent) { + let config = {}; + if (process.argv.indexOf('--unpinned') > -1) { config.unpinned = true; } + const ot = process.argv.indexOf('--olderthan'); + config.olderthan = ot > -1 && new Date(process.argv[ot+1]); + const bot = process.argv.indexOf('--blobsolderthan'); + config.blobsolderthan = bot > -1 && new Date(process.argv[bot+1]); + module.exports.load(config, function (err, data) { + if (err) { throw new Error(err); } + if (!Array.isArray(data)) { return; } + if (config.unpinned) { + data.forEach((f) => { console.log(f.filename + " " + f.size + " " + (+f.atime)); }); + } else { + data.forEach((x) => { console.log(x[0] + ' ' + x[1] + ' MB'); }); + } + }); +} diff --git a/www/common/common-interface.js b/www/common/common-interface.js index d0c0e1961..4822e357b 100644 --- a/www/common/common-interface.js +++ b/www/common/common-interface.js @@ -555,8 +555,11 @@ define([ $loading = $('#' + LOADING); //.show(); $loading.css('display', ''); $loading.removeClass('cp-loading-hidden'); + $('.cp-loading-spinner-container').show(); if (loadingText) { $('#' + LOADING).find('p').text(loadingText); + } else { + $('#' + LOADING).find('p').text(''); } $container = $loading.find('.cp-loading-container'); } else { @@ -612,7 +615,10 @@ define([ if (exitable) { $(window).focus(); $(window).keydown(function (e) { - if (e.which === 27) { $('#' + LOADING).hide(); } + if (e.which === 27) { + $('#' + LOADING).hide(); + if (typeof(exitable) === "function") { exitable(); } + } }); } }; diff --git a/www/common/sframe-app-framework.js b/www/common/sframe-app-framework.js index 7540f4a0b..d612387fa 100644 --- a/www/common/sframe-app-framework.js +++ b/www/common/sframe-app-framework.js @@ -426,15 +426,7 @@ define([ common.getSframeChannel().onReady(waitFor()); }).nThen(function (waitFor) { Test.registerInner(common.getSframeChannel()); - if (!AppConfig.displayCreationScreen) { return; } - var priv = common.getMetadataMgr().getPrivateData(); - if (priv.isNewFile) { - var c = (priv.settings.general && priv.settings.general.creation) || {}; - if (c.skip && !priv.forceCreationScreen) { - return void common.createPad(c, waitFor()); - } - common.getPadCreationScreen(c, waitFor()); - } + common.handleNewFile(waitFor); }).nThen(function (waitFor) { cpNfInner = common.startRealtime({ // really basic operational transform diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index 260efeda9..10fd2e9f9 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -187,7 +187,7 @@ define([ upgradeURL: Cryptpad.upgradeURL }, isNewFile: isNewFile, - isDeleted: window.location.hash.length > 0, + isDeleted: isNewFile && window.location.hash.length > 0, forceCreationScreen: forceCreationScreen }; for (var k in additionalPriv) { metaObj.priv[k] = additionalPriv[k]; } @@ -666,8 +666,9 @@ define([ Utils.Feedback.reportAppUsage(); if (!realtime) { return; } - if (isNewFile && Utils.LocalStore.isLoggedIn() - && AppConfig.displayCreationScreen && cfg.useCreationScreen) { return; } + if (isNewFile && cfg.useCreationScreen) { return; } + //if (isNewFile && Utils.LocalStore.isLoggedIn() + // && AppConfig.displayCreationScreen && cfg.useCreationScreen) { return; } startRealtime(); }); diff --git a/www/common/sframe-common.js b/www/common/sframe-common.js index 3f4070eaf..df6f77e87 100644 --- a/www/common/sframe-common.js +++ b/www/common/sframe-common.js @@ -167,6 +167,30 @@ define([ }; // Store + funcs.handleNewFile = function (waitFor) { + var priv = ctx.metadataMgr.getPrivateData(); + if (priv.isNewFile) { + var c = (priv.settings.general && priv.settings.general.creation) || {}; + var skip = !AppConfig.displayCreationScreen || (c.skip && !priv.forceCreationScreen); + // If this is a new file but we have a hash in the URL and pad creation screen is + // not displayed, then display an error... + if (priv.isDeleted && (!funcs.isLoggedIn() || skip)) { + UI.errorLoadingScreen(Messages.inactiveError, false, function () { + UI.addLoadingScreen(); + return void funcs.createPad({}, waitFor()); + }); + return; + } + // Otherwise, if we don't display the screen, it means it is not a deleted pad + // so we can continue and start realtime... + if (!funcs.isLoggedIn() || skip) { + return void funcs.createPad(c, waitFor()); + } + // If we display the pad creation screen, it will handle deleted pads directly + console.log('here'); + funcs.getPadCreationScreen(c, waitFor()); + } + }; funcs.createPad = function (cfg, cb) { ctx.sframeChan.query("Q_CREATE_PAD", { owned: cfg.owned, diff --git a/www/common/toolbar3.js b/www/common/toolbar3.js index e014503c6..df97cf072 100644 --- a/www/common/toolbar3.js +++ b/www/common/toolbar3.js @@ -593,7 +593,7 @@ define([ }; var createUnpinnedWarning0 = function (toolbar, config) { - if (true) { return; } // stub this call since it won't make it into the next release + //if (true) { return; } // stub this call since it won't make it into the next release if (Common.isLoggedIn()) { return; } var pd = config.metadataMgr.getPrivateData(); var o = pd.origin; diff --git a/www/poll/inner.js b/www/poll/inner.js index 8f31116a1..94d732a7f 100644 --- a/www/poll/inner.js +++ b/www/poll/inner.js @@ -1274,16 +1274,9 @@ define([ }).nThen(function (waitFor) { common.getSframeChannel().onReady(waitFor()); }).nThen(function (waitFor) { - if (!AppConfig.displayCreationScreen) { return; } - var priv = common.getMetadataMgr().getPrivateData(); - if (priv.isNewFile) { - var c = (priv.settings.general && priv.settings.general.creation) || {}; - if (c.skip && !priv.forceCreationScreen) { - return void common.createPad(c, waitFor()); - } - common.getPadCreationScreen(c, waitFor()); - } + common.handleNewFile(waitFor); }).nThen(function (/* waitFor */) { + console.log('here'); Test.registerInner(common.getSframeChannel()); var metadataMgr = common.getMetadataMgr(); APP.locked = APP.readOnly = metadataMgr.getPrivateData().readOnly; diff --git a/www/whiteboard/inner.js b/www/whiteboard/inner.js index 8d2621aaf..3c540fe98 100644 --- a/www/whiteboard/inner.js +++ b/www/whiteboard/inner.js @@ -662,15 +662,7 @@ define([ }).nThen(function (waitFor) { common.getSframeChannel().onReady(waitFor()); }).nThen(function (waitFor) { - if (!AppConfig.displayCreationScreen) { return; } - var priv = common.getMetadataMgr().getPrivateData(); - if (priv.isNewFile) { - var c = (priv.settings.general && priv.settings.general.creation) || {}; - if (c.skip && !priv.forceCreationScreen) { - return void common.createPad(c, waitFor()); - } - common.getPadCreationScreen(c, waitFor()); - } + common.handleNewFile(waitFor); }).nThen(function (/*waitFor*/) { andThen(common); });