disable automatic blob expiration since we can't quite rely on atime as an inactivity metric

package.json

@@ -41,6 +41,7 @@
     "flow": "./node_modules/.bin/flow",
     "test": "node scripts/TestSelenium.js",
     "test-rpc": "cd scripts/tests && node test-rpc",
-    "template": "cd customize.dist/src && for page in ../index.html ../privacy.html ../terms.html ../about.html ../contact.html ../what-is-cryptpad.html ../features.html ../../www/login/index.html ../../www/register/index.html ../../www/user/index.html;do echo $page; cp template.html $page; done;"
+    "template": "cd customize.dist/src && for page in ../index.html ../privacy.html ../terms.html ../about.html ../contact.html ../what-is-cryptpad.html ../features.html ../../www/login/index.html ../../www/register/index.html ../../www/user/index.html;do echo $page; cp template.html $page; done;",
+    "evict-inactive": "node scripts/evict-inactive.js"
   }
 }

scripts/evict-inactive.js

@@ -1,6 +1,7 @@
 var nThen = require("nthen");
 
 var Store = require("../storage/file");
+var BlobStore = require("../storage/blob");
 var Pinned = require("./pinned");
 var config = require("../lib/load-config");
 
@@ -14,9 +15,16 @@ var inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000);
 // files which were archived before this date can be considered safe to remove
 var retentionTime = +new Date() - (config.archiveRetentionTime * 24 * 3600 * 1000);
 
+var getNewestTime = function (stats) {
+    return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) {
+        return stats[b] > stats[a]? b: a;
+    })];
+};
+
 var store;
 var pins;
 var Log;
+var blobs;
 nThen(function (w) {
     // load the store which will be used for iterating over channels
     // and performing operations like archival and deletion
@@ -40,8 +48,17 @@ nThen(function (w) {
     Logger.create(config, w(function (_) {
         Log = _;
     }));
+
+    config.getSession = function () {};
+    BlobStore.create(config, w(function (err, _) {
+        if (err) {
+            w.abort();
+            return console.error(err);
+        }
+        blobs = _;
+    }));
 }).nThen(function (w) {
-    // this block will iterate over archived channels and remove them
+    // this block will iterate over archived channels and removes them
     // if they've been in cold storage for longer than your configured archive time
 
     // if the admin has not set an 'archiveRetentionTime', this block makes no sense
@@ -89,6 +106,104 @@ nThen(function (w) {
     };
 
     store.listArchivedChannels(handler, w(done));
+}).nThen(function (w) {
+    if (typeof(config.archiveRetentionTime) !== "number") { return; }
+    var removed = 0;
+    blobs.list.archived.proofs(function (err, item, next) {
+        if (err) {
+            Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err);
+            return void next();
+        }
+        if (pins[item.blobId]) { return void next(); }
+        if (item && getNewestTime(item) > retentionTime) { return void next(); }
+        blobs.remove.archived.proof(item.safeKey, item.blobId, (function (err) {
+            if (err) {
+                Log.error("EVICT_ARCHIVED_BLOB_PROOF_ERROR", item);
+                return void next();
+            }
+            Log.info("EVICT_ARCHIVED_BLOB_PROOF", item);
+            removed++;
+            next();
+        }));
+    }, w(function () {
+        Log.info('EVICT_ARCHIVED_BLOB_PROOFS_REMOVED', removed);
+    }));
+}).nThen(function (w) {
+    if (typeof(config.archiveRetentionTime) !== "number") { return; }
+    var removed = 0;
+    blobs.list.archived.blobs(function (err, item, next) {
+        if (err) {
+            Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err);
+            return void next();
+        }
+        if (pins[item.blobId]) { return void next(); }
+        if (item && getNewestTime(item) > retentionTime) { return void next(); }
+        blobs.remove.archived.blob(item.blobId, function (err) {
+            if (err) {
+                Log.error("EVICT_ARCHIVED_BLOB_ERROR", item);
+                return void next();
+            }
+            Log.info("EVICT_ARCHIVED_BLOB", item);
+            removed++;
+            next();
+        });
+    }, w(function () {
+        Log.info('EVICT_ARCHIVED_BLOBS_REMOVED', removed);
+    }));
+/*  TODO find a reliable metric for determining the activity of blobs...
+}).nThen(function (w) {
+    var blobCount = 0;
+    var lastHour = 0;
+    blobs.list.blobs(function (err, item, next) {
+        blobCount++;
+        if (err) {
+            Log.error("EVICT_BLOB_LIST_BLOBS_ERROR", err);
+            return void next();
+        }
+        if (pins[item.blobId]) { return void next(); }
+        if (item && getNewestTime(item) > retentionTime) { return void next(); }
+        // TODO determine when to retire blobs
+        console.log(item);
+        next();
+    }, w(function () {
+        console.log("Listed %s blobs", blobCount);
+        console.log("Listed %s blobs accessed in the last hour", lastHour);
+    }));
+}).nThen(function (w) {
+    var proofCount = 0;
+    blobs.list.proofs(function (err, item, next) {
+        proofCount++;
+        if (err) {
+            next();
+            return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err);
+        }
+        if (pins[item.blobId]) { return void next(); }
+        if (item && getNewestTime(item) > retentionTime) { return void next(); }
+        nThen(function (w) {
+            blobs.size(item.blobId, w(function (err, size) {
+                if (err) {
+                    w.abort();
+                    next();
+                    return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err);
+                }
+                if (size !== 0) {
+                    w.abort();
+                    next();
+                }
+            }));
+        }).nThen(function () {
+            blobs.remove.proof(item.safeKey, item.blobId, function (err) {
+                next();
+                if (err) {
+                    return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item);
+                }
+                return Log.info("EVICT_BLOB_PROOF_LONELY", item);
+            });
+        });
+    }, function () {
+        console.log("Listed %s blob proofs", proofCount);
+    });
+*/
 }).nThen(function (w) {
     var removed = 0;
     var channels = 0;

storage/blob.js

@@ -37,6 +37,14 @@ var makeProofPath = function (Env, safeKey, blobId) {
     return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId);
 };
 
+var parseProofPath = function (path) {
+    var parts = path.split('/');
+    return {
+        blobId: parts[parts.length -1],
+        safeKey: parts[parts.length - 3],
+    };
+};
+
 // getUploadSize: used by
     // getFileSize
 var getUploadSize = function (Env, blobId, cb) {
@@ -346,11 +354,11 @@ var restoreProof = function (Env, safeKey, blobId, cb) {
     Fse.move(archivePath, proofPath, cb);
 };
 
-var makeWalker = function (n, handleChild, cb) {
+var makeWalker = function (n, handleChild, done) {
     if (!n || typeof(n) !== 'number' || n < 2) { n = 2; }
 
     var W;
-    var nt = nThen(function (w) {
+    nThen(function (w) {
         // this asynchronous bit defers the completion of this block until
         // synchronous execution has completed. This means you must create
         // the walker and start using it synchronously or else it will call back
@@ -358,7 +366,7 @@ var makeWalker = function (n, handleChild, cb) {
         setTimeout(w());
         W = w;
     }).nThen(function () {
-        cb();
+        done();
     });
 
     // do no more than 20 jobs at a time
@@ -366,21 +374,28 @@ var makeWalker = function (n, handleChild, cb) {
 
     var recurse = function (path) {
         tasks.take(function (give) {
-            var done = give(W());
-            Fs.readdir(path, function (err, dir) {
-                if (err) {
-                    if (err.code === 'ENOTDIR') {
-                        return void handleChild(path, done);
-                    }
-                    // XXX handle other error
-                    return done();
+            var next = give(W());
+
+            nThen(function (w) {
+                // check if the path is a directory...
+                Fs.stat(path, w(function (err, stats) {
+                    if (err) { return next(); }
+                    if (!stats.isDirectory()) {
+                        w.abort();
+                        return void handleChild(void 0, path, next);
                     }
+                    // fall through
+                }));
+            }).nThen(function () {
+                // handle directories
+                Fs.readdir(path, function (err, dir) {
+                    if (err) { return next(); }
                     // everything is fine and it's a directory...
-                if (dir.length === 0) { return done(); }
                     dir.forEach(function (d) {
                         recurse(Path.join(path, d));
                     });
-                done();
+                    next();
+                });
             });
         });
     };
@@ -392,17 +407,28 @@ var listProofs = function (root, handler, cb) {
     Fs.readdir(root, function (err, dir) {
         if (err) { return void cb(err); }
 
-        var walk = makeWalker(20, function (path, next) {
+        var walk = makeWalker(20, function (err, path, next) {
             // path is the path to a child node on the filesystem
 
             // next handles the next job in a queue
 
                 // iterate over proofs
                 // check for presence of corresponding files
+            Fs.stat(path, function (err, stats) {
+                if (err) {
+                    return void handler(err, void 0, next);
+                }
 
-            handler(path, next);
-            //console.log(path);
-            //next();
+                var parsed = parseProofPath(path);
+                handler(void 0, {
+                    path: path,
+                    blobId: parsed.blobId,
+                    safeKey: parsed.safeKey,
+                    atime: stats.atime,
+                    ctime: stats.ctime,
+                    mtime: stats.mtime,
+                }, next);
+            });
         }, function () {
             // called when there are no more directories or children to process
             cb();
@@ -420,8 +446,19 @@ var listBlobs = function (root, handler, cb) {
     // iterate over files
     Fs.readdir(root, function (err, dir) {
         if (err) { return void cb(err); }
-        var walk = makeWalker(20, function (path, next) {
-            handler(path, next);
+        var walk = makeWalker(20, function (err, path, next) {
+            Fs.stat(path, function (err, stats) {
+                if (err) {
+                    return void handler(err, void 0, next);
+                }
+
+                handler(void 0, {
+                    blobId: Path.basename(path),
+                    atime: stats.atime,
+                    ctime: stats.ctime,
+                    mtime: stats.mtime,
+                }, next);
+            });
         }, function () {
             cb();
         });