diff --git a/package.json b/package.json index fdbbc008a..2997cc985 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "flow": "./node_modules/.bin/flow", "test": "node scripts/TestSelenium.js", "test-rpc": "cd scripts/tests && node test-rpc", - "template": "cd customize.dist/src && for page in ../index.html ../privacy.html ../terms.html ../about.html ../contact.html ../what-is-cryptpad.html ../features.html ../../www/login/index.html ../../www/register/index.html ../../www/user/index.html;do echo $page; cp template.html $page; done;" + "template": "cd customize.dist/src && for page in ../index.html ../privacy.html ../terms.html ../about.html ../contact.html ../what-is-cryptpad.html ../features.html ../../www/login/index.html ../../www/register/index.html ../../www/user/index.html;do echo $page; cp template.html $page; done;", + "evict-inactive": "node scripts/evict-inactive.js" } } diff --git a/scripts/evict-inactive.js b/scripts/evict-inactive.js index ff9a3c343..ebb65d637 100644 --- a/scripts/evict-inactive.js +++ b/scripts/evict-inactive.js @@ -1,6 +1,7 @@ var nThen = require("nthen"); var Store = require("../storage/file"); +var BlobStore = require("../storage/blob"); var Pinned = require("./pinned"); var config = require("../lib/load-config"); @@ -14,9 +15,16 @@ var inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000); // files which were archived before this date can be considered safe to remove var retentionTime = +new Date() - (config.archiveRetentionTime * 24 * 3600 * 1000); +var getNewestTime = function (stats) { + return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) { + return stats[b] > stats[a]? b: a; + })]; +}; + var store; var pins; var Log; +var blobs; nThen(function (w) { // load the store which will be used for iterating over channels // and performing operations like archival and deletion @@ -40,8 +48,17 @@ nThen(function (w) { Logger.create(config, w(function (_) { Log = _; })); + + config.getSession = function () {}; + BlobStore.create(config, w(function (err, _) { + if (err) { + w.abort(); + return console.error(err); + } + blobs = _; + })); }).nThen(function (w) { - // this block will iterate over archived channels and remove them + // this block will iterate over archived channels and removes them // if they've been in cold storage for longer than your configured archive time // if the admin has not set an 'archiveRetentionTime', this block makes no sense @@ -89,6 +106,104 @@ nThen(function (w) { }; store.listArchivedChannels(handler, w(done)); +}).nThen(function (w) { + if (typeof(config.archiveRetentionTime) !== "number") { return; } + var removed = 0; + blobs.list.archived.proofs(function (err, item, next) { + if (err) { + Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err); + return void next(); + } + if (pins[item.blobId]) { return void next(); } + if (item && getNewestTime(item) > retentionTime) { return void next(); } + blobs.remove.archived.proof(item.safeKey, item.blobId, (function (err) { + if (err) { + Log.error("EVICT_ARCHIVED_BLOB_PROOF_ERROR", item); + return void next(); + } + Log.info("EVICT_ARCHIVED_BLOB_PROOF", item); + removed++; + next(); + })); + }, w(function () { + Log.info('EVICT_ARCHIVED_BLOB_PROOFS_REMOVED', removed); + })); +}).nThen(function (w) { + if (typeof(config.archiveRetentionTime) !== "number") { return; } + var removed = 0; + blobs.list.archived.blobs(function (err, item, next) { + if (err) { + Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err); + return void next(); + } + if (pins[item.blobId]) { return void next(); } + if (item && getNewestTime(item) > retentionTime) { return void next(); } + blobs.remove.archived.blob(item.blobId, function (err) { + if (err) { + Log.error("EVICT_ARCHIVED_BLOB_ERROR", item); + return void next(); + } + Log.info("EVICT_ARCHIVED_BLOB", item); + removed++; + next(); + }); + }, w(function () { + Log.info('EVICT_ARCHIVED_BLOBS_REMOVED', removed); + })); +/* TODO find a reliable metric for determining the activity of blobs... +}).nThen(function (w) { + var blobCount = 0; + var lastHour = 0; + blobs.list.blobs(function (err, item, next) { + blobCount++; + if (err) { + Log.error("EVICT_BLOB_LIST_BLOBS_ERROR", err); + return void next(); + } + if (pins[item.blobId]) { return void next(); } + if (item && getNewestTime(item) > retentionTime) { return void next(); } + // TODO determine when to retire blobs + console.log(item); + next(); + }, w(function () { + console.log("Listed %s blobs", blobCount); + console.log("Listed %s blobs accessed in the last hour", lastHour); + })); +}).nThen(function (w) { + var proofCount = 0; + blobs.list.proofs(function (err, item, next) { + proofCount++; + if (err) { + next(); + return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err); + } + if (pins[item.blobId]) { return void next(); } + if (item && getNewestTime(item) > retentionTime) { return void next(); } + nThen(function (w) { + blobs.size(item.blobId, w(function (err, size) { + if (err) { + w.abort(); + next(); + return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err); + } + if (size !== 0) { + w.abort(); + next(); + } + })); + }).nThen(function () { + blobs.remove.proof(item.safeKey, item.blobId, function (err) { + next(); + if (err) { + return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item); + } + return Log.info("EVICT_BLOB_PROOF_LONELY", item); + }); + }); + }, function () { + console.log("Listed %s blob proofs", proofCount); + }); +*/ }).nThen(function (w) { var removed = 0; var channels = 0; diff --git a/storage/blob.js b/storage/blob.js index fd39a7f67..6b3e891dd 100644 --- a/storage/blob.js +++ b/storage/blob.js @@ -37,6 +37,14 @@ var makeProofPath = function (Env, safeKey, blobId) { return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId); }; +var parseProofPath = function (path) { + var parts = path.split('/'); + return { + blobId: parts[parts.length -1], + safeKey: parts[parts.length - 3], + }; +}; + // getUploadSize: used by // getFileSize var getUploadSize = function (Env, blobId, cb) { @@ -346,11 +354,11 @@ var restoreProof = function (Env, safeKey, blobId, cb) { Fse.move(archivePath, proofPath, cb); }; -var makeWalker = function (n, handleChild, cb) { +var makeWalker = function (n, handleChild, done) { if (!n || typeof(n) !== 'number' || n < 2) { n = 2; } var W; - var nt = nThen(function (w) { + nThen(function (w) { // this asynchronous bit defers the completion of this block until // synchronous execution has completed. This means you must create // the walker and start using it synchronously or else it will call back @@ -358,7 +366,7 @@ var makeWalker = function (n, handleChild, cb) { setTimeout(w()); W = w; }).nThen(function () { - cb(); + done(); }); // do no more than 20 jobs at a time @@ -366,21 +374,28 @@ var makeWalker = function (n, handleChild, cb) { var recurse = function (path) { tasks.take(function (give) { - var done = give(W()); - Fs.readdir(path, function (err, dir) { - if (err) { - if (err.code === 'ENOTDIR') { - return void handleChild(path, done); + var next = give(W()); + + nThen(function (w) { + // check if the path is a directory... + Fs.stat(path, w(function (err, stats) { + if (err) { return next(); } + if (!stats.isDirectory()) { + w.abort(); + return void handleChild(void 0, path, next); } - // XXX handle other error - return done(); - } - // everything is fine and it's a directory... - if (dir.length === 0) { return done(); } - dir.forEach(function (d) { - recurse(Path.join(path, d)); + // fall through + })); + }).nThen(function () { + // handle directories + Fs.readdir(path, function (err, dir) { + if (err) { return next(); } + // everything is fine and it's a directory... + dir.forEach(function (d) { + recurse(Path.join(path, d)); + }); + next(); }); - done(); }); }); }; @@ -392,17 +407,28 @@ var listProofs = function (root, handler, cb) { Fs.readdir(root, function (err, dir) { if (err) { return void cb(err); } - var walk = makeWalker(20, function (path, next) { + var walk = makeWalker(20, function (err, path, next) { // path is the path to a child node on the filesystem // next handles the next job in a queue // iterate over proofs // check for presence of corresponding files + Fs.stat(path, function (err, stats) { + if (err) { + return void handler(err, void 0, next); + } - handler(path, next); - //console.log(path); - //next(); + var parsed = parseProofPath(path); + handler(void 0, { + path: path, + blobId: parsed.blobId, + safeKey: parsed.safeKey, + atime: stats.atime, + ctime: stats.ctime, + mtime: stats.mtime, + }, next); + }); }, function () { // called when there are no more directories or children to process cb(); @@ -420,8 +446,19 @@ var listBlobs = function (root, handler, cb) { // iterate over files Fs.readdir(root, function (err, dir) { if (err) { return void cb(err); } - var walk = makeWalker(20, function (path, next) { - handler(path, next); + var walk = makeWalker(20, function (err, path, next) { + Fs.stat(path, function (err, stats) { + if (err) { + return void handler(err, void 0, next); + } + + handler(void 0, { + blobId: Path.basename(path), + atime: stats.atime, + ctime: stats.ctime, + mtime: stats.mtime, + }, next); + }); }, function () { cb(); });