From c388641479128303363d8a4247f64230c08a7264 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 23 Jan 2020 15:13:19 -0500 Subject: [PATCH] drop support for 'retainData' configuration --- config/config.example.js | 21 +++++---------- lib/historyKeeper.js | 16 +++-------- lib/rpc.js | 57 ++++++--------------------------------- scripts/evict-inactive.js | 50 ++++------------------------------ server.js | 1 - storage/file.js | 4 --- storage/tasks.js | 17 ------------ 7 files changed, 22 insertions(+), 144 deletions(-) diff --git a/config/config.example.js b/config/config.example.js index 9981c0626..90e96a66a 100644 --- a/config/config.example.js +++ b/config/config.example.js @@ -231,26 +231,17 @@ module.exports = { */ inactiveTime: 90, // days - /* CryptPad can be configured to remove inactive data which has not been pinned. - * Deletion of data is always risky and as an operator you have the choice to - * archive data instead of deleting it outright. Set this value to true if - * you want your server to archive files and false if you want to keep using - * the old behaviour of simply removing files. + /* CryptPad archives some data instead of deleting it outright. + * This archived data still takes up space and so you'll probably still want to + * remove these files after a brief period. + * + * cryptpad/scripts/evict-inactive.js is intended to be run daily + * from a crontab or similar scheduling service. * - * WARNING: this is not implemented universally, so at the moment this will - * only apply to the removal of 'channels' due to inactivity. - */ - retainData: true, - - /* As described above, CryptPad offers the ability to archive some data - * instead of deleting it outright. This archived data still takes up space - * and so you'll probably still want to remove these files after a brief period. * The intent with this feature is to provide a safety net in case of accidental * deletion. Set this value to the number of days you'd like to retain * archived data before it's removed permanently. * - * If 'retainData' is set to false, there will never be any archived data - * to remove. */ archiveRetentionTime: 15, diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index 414d064c3..09f5fdfea 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -83,7 +83,6 @@ module.exports.create = function (cfg) { const rpc = cfg.rpc; const tasks = cfg.tasks; const store = cfg.store; - const retainData = cfg.retainData; Log = cfg.log; Log.silly('HK_LOADING', 'LOADING HISTORY_KEEPER MODULE'); @@ -350,18 +349,9 @@ module.exports.create = function (cfg) { but for some reason are still present */ const expireChannel = function (ctx, channel) { - if (retainData) { - return void store.archiveChannel(channel, function (err) { - Log.info("ARCHIVAL_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", { - channelId: channel, - status: err? String(err): "SUCCESS", - }); - }); - } - - store.removeChannel(channel, function (err) { - Log.info("DELETION_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", { - channelid: channel, + return void store.archiveChannel(channel, function (err) { + Log.info("ARCHIVAL_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", { + channelId: channel, status: err? String(err): "SUCCESS", }); }); diff --git a/lib/rpc.js b/lib/rpc.js index dfe27302d..0ece7915c 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -878,22 +878,8 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { })); }).nThen(function (w) { // remove the blob - - if (Env.retainData) { - return void Env.blobStore.archive.blob(blobId, w(function (err) { - Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', { - safeKey: safeKey, - blobId: blobId, - status: err? String(err): 'SUCCESS', - }); - if (err) { - w.abort(); - return void cb(err); - } - })); - } - Env.blobStore.remove.blob(blobId, w(function (err) { - Log.info('DELETION_OWNED_FILE_BY_OWNER_RPC', { + return void Env.blobStore.archive.blob(blobId, w(function (err) { + Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', { safeKey: safeKey, blobId: blobId, status: err? String(err): 'SUCCESS', @@ -904,23 +890,9 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { } })); }).nThen(function () { - // remove the proof - if (Env.retainData) { - return void Env.blobStore.archive.proof(safeKey, blobId, function (err) { - Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", { - safeKey: safeKey, - blobId: blobId, - status: err? String(err): 'SUCCESS', - }); - if (err) { - return void cb("E_PROOF_REMOVAL"); - } - cb(); - }); - } - - Env.blobStore.remove.proof(safeKey, blobId, function (err) { - Log.info("DELETION_PROOF_REMOVAL_BY_OWNER_RPC", { + // archive the proof + return void Env.blobStore.archive.proof(safeKey, blobId, function (err) { + Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", { safeKey: safeKey, blobId: blobId, status: err? String(err): 'SUCCESS', @@ -939,21 +911,9 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { if (!isOwner(metadata, unsafeKey)) { return void cb('INSUFFICIENT_PERMISSIONS'); } - // if the admin has configured data retention... - // temporarily archive the file instead of removing it - if (Env.retainData) { - return void Env.msgStore.archiveChannel(channelId, function (e) { - Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', { - unsafeKey: unsafeKey, - channelId: channelId, - status: e? String(e): 'SUCCESS', - }); - cb(e); - }); - } - - return void Env.msgStore.removeChannel(channelId, function (e) { - Log.info('DELETION_CHANNEL_BY_OWNER_RPC', { + // temporarily archive the file + return void Env.msgStore.archiveChannel(channelId, function (e) { + Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', { unsafeKey: unsafeKey, channelId: channelId, status: e? String(e): 'SUCCESS', @@ -1437,7 +1397,6 @@ RPC.create = function (config, cb) { }; var Env = { - retainData: config.retainData || false, defaultStorageLimit: config.defaultStorageLimit, maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024), Sessions: {}, diff --git a/scripts/evict-inactive.js b/scripts/evict-inactive.js index 13028b8ff..f0e801909 100644 --- a/scripts/evict-inactive.js +++ b/scripts/evict-inactive.js @@ -15,8 +15,6 @@ var inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000); // files which were archived before this date can be considered safe to remove var retentionTime = +new Date() - (config.archiveRetentionTime * 24 * 3600 * 1000); -var retainData = Boolean(config.retainData); - var getNewestTime = function (stats) { return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) { return stats[b] > stats[a]? b: a; @@ -176,23 +174,6 @@ nThen(function (w) { if (pins[item.blobId]) { return void next(); } if (item && getNewestTime(item) > retentionTime) { return void next(); } - if (!retainData) { - return void blobs.remove.blob(item.blobId, function (err) { - if (err) { - Log.error("EVICT_BLOB_ERROR", { - error: err, - item: item, - }); - return void next(); - } - Log.info("EVICT_BLOB_INACTIVE", { - item: item, - }); - removed++; - next(); - }); - } - blobs.archive.blob(item.blobId, function (err) { if (err) { Log.error("EVICT_ARCHIVE_BLOB_ERROR", { @@ -247,7 +228,6 @@ nThen(function (w) { Log.info("EVICT_BLOB_PROOFS_REMOVED", removed); })); }).nThen(function (w) { - var removed = 0; var channels = 0; var archived = 0; @@ -279,42 +259,22 @@ nThen(function (w) { // ignore the channel if it's pinned if (pins[item.channel]) { return void cb(); } - // if the server is configured to retain data, archive the channel - if (config.retainData) { - return void store.archiveChannel(item.channel, w(function (err) { - if (err) { - Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', { - error: err, - channel: item.channel, - }); - return void cb(); - } - Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel); - archived++; - cb(); - })); - } - - // otherwise remove it - store.removeChannel(item.channel, w(function (err) { + return void store.archiveChannel(item.channel, w(function (err) { if (err) { - Log.error('EVICT_CHANNEL_REMOVAL_ERROR', { + Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', { error: err, channel: item.channel, }); return void cb(); } - Log.info('EVICT_CHANNEL_REMOVAL', item.channel); - removed++; + Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel); + archived++; cb(); })); }; var done = function () { - if (config.retainData) { - return void Log.info('EVICT_CHANNELS_ARCHIVED', archived); - } - return void Log.info('EVICT_CHANNELS_REMOVED', removed); + return void Log.info('EVICT_CHANNELS_ARCHIVED', archived); }; store.listChannels(handler, w(done)); diff --git a/server.js b/server.js index fec59a7ea..d2bdcabd8 100644 --- a/server.js +++ b/server.js @@ -278,7 +278,6 @@ var nt = nThen(function (w) { rpc: rpc, store: config.store, log: log, - retainData: Boolean(config.retainData), }; historyKeeper = HK.create(hkConfig); }).nThen(function () { diff --git a/storage/file.js b/storage/file.js index b2899619f..a3d6f521f 100644 --- a/storage/file.js +++ b/storage/file.js @@ -555,9 +555,6 @@ var listChannels = function (root, handler, cb) { // to an equivalent location in the cold storage directory var archiveChannel = function (env, channelName, cb) { // TODO close channels before archiving them? - if (!env.retainData) { - return void cb("ARCHIVES_DISABLED"); - } // ctime is the most reliable indicator of when a file was archived // because it is used to indicate changes to the files metadata @@ -875,7 +872,6 @@ module.exports.create = function (conf, cb) { var env = { root: conf.filePath || './datastore', archiveRoot: conf.archivePath || './data/archive', - retainData: conf.retainData, channels: { }, channelExpirationMs: conf.channelExpirationMs || 30000, verbose: conf.verbose, diff --git a/storage/tasks.js b/storage/tasks.js index 2209b3d59..bb4dbdb9c 100644 --- a/storage/tasks.js +++ b/storage/tasks.js @@ -202,22 +202,6 @@ var expire = function (env, task, cb) { var Log = env.log; var args = task.slice(2); - if (!env.retainData) { - Log.info('DELETION_SCHEDULED_EXPIRATION', { - task: task, - }); - env.store.removeChannel(args[0], function (err) { - if (err) { - Log.error('DELETION_SCHEDULED_EXPIRATION_ERROR', { - task: task, - error: err, - }); - } - cb(); - }); - return; - } - Log.info('ARCHIVAL_SCHEDULED_EXPIRATION', { task: task, }); @@ -381,7 +365,6 @@ Tasks.create = function (config, cb) { root: config.taskPath || './tasks', log: config.log, store: config.store, - retainData: Boolean(config.retainData), }; // make sure the path exists...