infra
/
cryptpad
2
1
Fork 0
Code Issues Pull Requests Projects Releases 1 Activity

evict unpinned blobs and blob proofs

pull/1/head
ansuz 2020-01-20 15:54:15 -05:00
parent 66c857d61e
commit e6d5b0886e
1 changed files with 49 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
scripts/evict-inactive.js
View File

@ -15,6 +15,8 @@ var inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000);
// files which were archived before this date can be considered safe to remove
var retentionTime = +new Date() - (config.archiveRetentionTime * 24 * 3600 * 1000);
var retainData = Boolean(config.retainData);
var getNewestTime = function (stats) {
return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) {
return stats[b] > stats[a]? b: a;
@ -116,6 +118,8 @@ nThen(function (w) {
store.listArchivedChannels(handler, w(done));
}).nThen(function (w) {
if (typeof(config.archiveRetentionTime) !== "number") { return; }
// Iterate over archive blob ownership proofs and remove them
// if they are older than the specified retention time
var removed = 0;
blobs.list.archived.proofs(function (err, item, next) {
if (err) {
@ -138,6 +142,8 @@ nThen(function (w) {
}));
}).nThen(function (w) {
if (typeof(config.archiveRetentionTime) !== "number") { return; }
// Iterate over archived blobs and remove them
// if they are older than the specified retention time
var removed = 0;
blobs.list.archived.blobs(function (err, item, next) {
if (err) {
@ -158,29 +164,57 @@ nThen(function (w) {
}, w(function () {
Log.info('EVICT_ARCHIVED_BLOBS_REMOVED', removed);
}));
/* TODO find a reliable metric for determining the activity of blobs...
}).nThen(function (w) {
var blobCount = 0;
var lastHour = 0;
// iterate over blobs and remove them
// if they have not been accessed within the specified retention time
var removed = 0;
blobs.list.blobs(function (err, item, next) {
blobCount++;
if (err) {
Log.error("EVICT_BLOB_LIST_BLOBS_ERROR", err);
return void next();
}
if (pins[item.blobId]) { return void next(); }
if (item && getNewestTime(item) > retentionTime) { return void next(); }
// TODO determine when to retire blobs
console.log(item);
next();
if (!retainData) {
return void blobs.remove.blob(item.blobId, function (err) {
if (err) {
Log.error("EVICT_BLOB_ERROR", {
error: err,
item: item,
});
return void next();
}
Log.info("EVICT_BLOB_INACTIVE", {
item: item,
});
removed++;
next();
});
}
blobs.archive.blob(item.blobId, function (err) {
if (err) {
Log.error("EVICT_ARCHIVE_BLOB_ERROR", {
error: err,
item: item,
});
return void next();
}
Log.info("EVICT_ARCHIVE_BLOB", {
item: item,
});
removed++;
next();
});
}, w(function () {
console.log("Listed %s blobs", blobCount);
console.log("Listed %s blobs accessed in the last hour", lastHour);
Log.info('EVICT_BLOBS_REMOVED', removed);
}));
}).nThen(function (w) {
var proofCount = 0;
// iterate over blob proofs and remove them
// if they don't correspond to a pinned or active file
var removed = 0;
blobs.list.proofs(function (err, item, next) {
proofCount++;
if (err) {
next();
return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err);
@ -205,13 +239,13 @@ nThen(function (w) {
if (err) {
return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item);
}
removed++;
return Log.info("EVICT_BLOB_PROOF_LONELY", item);
});
});
}, function () {
console.log("Listed %s blob proofs", proofCount);
});
*/
}, w(function () {
Log.info("EVICT_BLOB_PROOFS_REMOVED", removed);
}));
}).nThen(function (w) {
var removed = 0;
var channels = 0;