infra
/
cryptpad
2
1
Fork 0
Code Issues Pull Requests Projects Releases 1 Activity

drop support for 'retainData' configuration

pull/1/head
ansuz 2020-01-23 15:13:19 -05:00
parent f45de2b52f
commit c388641479
7 changed files with 22 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
config/config.example.js
View File

@ -231,26 +231,17 @@ module.exports = {
*/
inactiveTime: 90, // days
/* CryptPad can be configured to remove inactive data which has not been pinned.
* Deletion of data is always risky and as an operator you have the choice to
* archive data instead of deleting it outright. Set this value to true if
* you want your server to archive files and false if you want to keep using
* the old behaviour of simply removing files.
/* CryptPad archives some data instead of deleting it outright.
* This archived data still takes up space and so you'll probably still want to
* remove these files after a brief period.
*
* cryptpad/scripts/evict-inactive.js is intended to be run daily
* from a crontab or similar scheduling service.
*
* WARNING: this is not implemented universally, so at the moment this will
* only apply to the removal of 'channels' due to inactivity.
*/
retainData: true,
/* As described above, CryptPad offers the ability to archive some data
* instead of deleting it outright. This archived data still takes up space
* and so you'll probably still want to remove these files after a brief period.
* The intent with this feature is to provide a safety net in case of accidental
* deletion. Set this value to the number of days you'd like to retain
* archived data before it's removed permanently.
*
* If 'retainData' is set to false, there will never be any archived data
* to remove.
*/
archiveRetentionTime: 15,

16
lib/historyKeeper.js
View File

@ -83,7 +83,6 @@ module.exports.create = function (cfg) {
const rpc = cfg.rpc;
const tasks = cfg.tasks;
const store = cfg.store;
const retainData = cfg.retainData;
Log = cfg.log;
Log.silly('HK_LOADING', 'LOADING HISTORY_KEEPER MODULE');
@ -350,18 +349,9 @@ module.exports.create = function (cfg) {
but for some reason are still present
*/
const expireChannel = function (ctx, channel) {
if (retainData) {
return void store.archiveChannel(channel, function (err) {
Log.info("ARCHIVAL_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", {
channelId: channel,
status: err? String(err): "SUCCESS",
});
});
}
store.removeChannel(channel, function (err) {
Log.info("DELETION_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", {
channelid: channel,
return void store.archiveChannel(channel, function (err) {
Log.info("ARCHIVAL_CHANNEL_BY_HISTORY_KEEPER_EXPIRATION", {
channelId: channel,
status: err? String(err): "SUCCESS",
});
});

57
lib/rpc.js
View File

@ -878,22 +878,8 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) {
}));
}).nThen(function (w) {
// remove the blob
if (Env.retainData) {
return void Env.blobStore.archive.blob(blobId, w(function (err) {
Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', {
safeKey: safeKey,
blobId: blobId,
status: err? String(err): 'SUCCESS',
});
if (err) {
w.abort();
return void cb(err);
}
}));
}
Env.blobStore.remove.blob(blobId, w(function (err) {
Log.info('DELETION_OWNED_FILE_BY_OWNER_RPC', {
return void Env.blobStore.archive.blob(blobId, w(function (err) {
Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', {
safeKey: safeKey,
blobId: blobId,
status: err? String(err): 'SUCCESS',
@ -904,23 +890,9 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) {
}
}));
}).nThen(function () {
// remove the proof
if (Env.retainData) {
return void Env.blobStore.archive.proof(safeKey, blobId, function (err) {
Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", {
safeKey: safeKey,
blobId: blobId,
status: err? String(err): 'SUCCESS',
});
if (err) {
return void cb("E_PROOF_REMOVAL");
}
cb();
});
}
Env.blobStore.remove.proof(safeKey, blobId, function (err) {
Log.info("DELETION_PROOF_REMOVAL_BY_OWNER_RPC", {
// archive the proof
return void Env.blobStore.archive.proof(safeKey, blobId, function (err) {
Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", {
safeKey: safeKey,
blobId: blobId,
status: err? String(err): 'SUCCESS',
@ -939,21 +911,9 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) {
if (!isOwner(metadata, unsafeKey)) {
return void cb('INSUFFICIENT_PERMISSIONS');
}
// if the admin has configured data retention...
// temporarily archive the file instead of removing it
if (Env.retainData) {
return void Env.msgStore.archiveChannel(channelId, function (e) {
Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', {
unsafeKey: unsafeKey,
channelId: channelId,
status: e? String(e): 'SUCCESS',
});
cb(e);
});
}
return void Env.msgStore.removeChannel(channelId, function (e) {
Log.info('DELETION_CHANNEL_BY_OWNER_RPC', {
// temporarily archive the file
return void Env.msgStore.archiveChannel(channelId, function (e) {
Log.info('ARCHIVAL_CHANNEL_BY_OWNER_RPC', {
unsafeKey: unsafeKey,
channelId: channelId,
status: e? String(e): 'SUCCESS',
@ -1437,7 +1397,6 @@ RPC.create = function (config, cb) {
};
var Env = {
retainData: config.retainData || false,
defaultStorageLimit: config.defaultStorageLimit,
maxUploadSize: config.maxUploadSize || (20 * 1024 * 1024),
Sessions: {},

50
scripts/evict-inactive.js
View File

@ -15,8 +15,6 @@ var inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000);
// files which were archived before this date can be considered safe to remove
var retentionTime = +new Date() - (config.archiveRetentionTime * 24 * 3600 * 1000);
var retainData = Boolean(config.retainData);
var getNewestTime = function (stats) {
return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) {
return stats[b] > stats[a]? b: a;
@ -176,23 +174,6 @@ nThen(function (w) {
if (pins[item.blobId]) { return void next(); }
if (item && getNewestTime(item) > retentionTime) { return void next(); }
if (!retainData) {
return void blobs.remove.blob(item.blobId, function (err) {
if (err) {
Log.error("EVICT_BLOB_ERROR", {
error: err,
item: item,
});
return void next();
}
Log.info("EVICT_BLOB_INACTIVE", {
item: item,
});
removed++;
next();
});
}
blobs.archive.blob(item.blobId, function (err) {
if (err) {
Log.error("EVICT_ARCHIVE_BLOB_ERROR", {
@ -247,7 +228,6 @@ nThen(function (w) {
Log.info("EVICT_BLOB_PROOFS_REMOVED", removed);
}));
}).nThen(function (w) {
var removed = 0;
var channels = 0;
var archived = 0;
@ -279,42 +259,22 @@ nThen(function (w) {
// ignore the channel if it's pinned
if (pins[item.channel]) { return void cb(); }
// if the server is configured to retain data, archive the channel
if (config.retainData) {
return void store.archiveChannel(item.channel, w(function (err) {
if (err) {
Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', {
error: err,
channel: item.channel,
});
return void cb();
}
Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel);
archived++;
cb();
}));
}
// otherwise remove it
store.removeChannel(item.channel, w(function (err) {
return void store.archiveChannel(item.channel, w(function (err) {
if (err) {
Log.error('EVICT_CHANNEL_REMOVAL_ERROR', {
Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', {
error: err,
channel: item.channel,
});
return void cb();
}
Log.info('EVICT_CHANNEL_REMOVAL', item.channel);
removed++;
Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel);
archived++;
cb();
}));
};
var done = function () {
if (config.retainData) {
return void Log.info('EVICT_CHANNELS_ARCHIVED', archived);
}
return void Log.info('EVICT_CHANNELS_REMOVED', removed);
return void Log.info('EVICT_CHANNELS_ARCHIVED', archived);
};
store.listChannels(handler, w(done));

1
server.js
View File

@ -278,7 +278,6 @@ var nt = nThen(function (w) {
rpc: rpc,
store: config.store,
log: log,
retainData: Boolean(config.retainData),
};
historyKeeper = HK.create(hkConfig);
}).nThen(function () {

4
storage/file.js
View File

@ -555,9 +555,6 @@ var listChannels = function (root, handler, cb) {
// to an equivalent location in the cold storage directory
var archiveChannel = function (env, channelName, cb) {
// TODO close channels before archiving them?
if (!env.retainData) {
return void cb("ARCHIVES_DISABLED");
}
// ctime is the most reliable indicator of when a file was archived
// because it is used to indicate changes to the files metadata
@ -875,7 +872,6 @@ module.exports.create = function (conf, cb) {
var env = {
root: conf.filePath || './datastore',
archiveRoot: conf.archivePath || './data/archive',
retainData: conf.retainData,
channels: { },
channelExpirationMs: conf.channelExpirationMs || 30000,
verbose: conf.verbose,

17
storage/tasks.js
View File

@ -202,22 +202,6 @@ var expire = function (env, task, cb) {
var Log = env.log;
var args = task.slice(2);
if (!env.retainData) {
Log.info('DELETION_SCHEDULED_EXPIRATION', {
task: task,
});
env.store.removeChannel(args[0], function (err) {
if (err) {
Log.error('DELETION_SCHEDULED_EXPIRATION_ERROR', {
task: task,
error: err,
});
}
cb();
});
return;
}
Log.info('ARCHIVAL_SCHEDULED_EXPIRATION', {
task: task,
});
@ -381,7 +365,6 @@ Tasks.create = function (config, cb) {
root: config.taskPath || './tasks',
log: config.log,
store: config.store,
retainData: Boolean(config.retainData),
};
// make sure the path exists...