archive inactive data in one of the server's workers

lib/eviction.js

@@ -15,14 +15,11 @@ Env = {
     limits: {
         <unsafeKey>: <limit>,
     },
-    config: {
+    archiveRetentionTime: <number of days>,
-        inactiveTime: <number of days>,
+    accountRetentionTime: <number of days>,
-        archiveRetentionTime: <number of days>,
+    inactiveTime: <number of days>,
-        accountRetentionTime: <number of days>,
+    paths: {
-
+        pin: <path to pin storage>
-        pinPath: <filesystem path>,
-
-        customLimits: <custom limits map>,
     },
     store,
     pinStore,
@@ -35,11 +32,9 @@ Env = {
 module.exports = function (Env, cb) {
     var complete = Util.once(Util.mkAsync(cb));
 
-    var config = Env.config;
-
     // the administrator should have set an 'inactiveTime' in their config
     // if they didn't, just exit.
-    if (!config.inactiveTime || typeof(config.inactiveTime) !== "number") {
+    if (!Env.inactiveTime || typeof(Env.inactiveTime) !== "number") {
         return void complete("NO_INACTIVE_TIME");
     }
 
@@ -55,10 +50,10 @@ module.exports = function (Env, cb) {
         });
 
     // files which have not been changed since before this date can be considered inactive
-    var inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000);
+    var inactiveTime = +new Date() - (Env.inactiveTime * 24 * 3600 * 1000);
 
     // files which were archived before this date can be considered safe to remove
-    var retentionTime = +new Date() - (config.archiveRetentionTime * 24 * 3600 * 1000);
+    var retentionTime = +new Date() - (Env.archiveRetentionTime * 24 * 3600 * 1000);
 
     var store;
     var pinStore;
@@ -111,7 +106,7 @@ module.exports = function (Env, cb) {
 
         // if the admin has not set an 'archiveRetentionTime', this block makes no sense
         // so just skip it
-        if (typeof(config.archiveRetentionTime) !== "number") { return; }
+        if (typeof(Env.archiveRetentionTime) !== "number") { return; }
 
         // count the number of files which have been removed in this run
         var removed = 0;
@@ -165,7 +160,7 @@ module.exports = function (Env, cb) {
     };
 
     var removeArchivedBlobProofs = function (w) {
-        if (typeof(config.archiveRetentionTime) !== "number") { return; }
+        if (typeof(Env.archiveRetentionTime) !== "number") { return; }
     // Iterate over archive blob ownership proofs and remove them
     // if they are older than the specified retention time
         var removed = 0;
@@ -190,7 +185,7 @@ module.exports = function (Env, cb) {
     };
 
     var removeArchivedBlobs = function (w) {
-        if (typeof(config.archiveRetentionTime) !== "number") { return; }
+        if (typeof(Env.archiveRetentionTime) !== "number") { return; }
     // Iterate over archived blobs and remove them
     // if they are older than the specified retention time
         var removed = 0;
@@ -280,8 +275,8 @@ module.exports = function (Env, cb) {
         var inactive = 0;
 
         var accountRetentionTime;
-        if (typeof(config.accountRetentionTime) === 'number' && config.accountRetentionTime > 0) {
+        if (typeof(Env.accountRetentionTime) === 'number' && Env.accountRetentionTime > 0) {
-            accountRetentionTime = +new Date() - (24 * 3600 * 1000 * config.accountRetentionTime);
+            accountRetentionTime = +new Date() - (24 * 3600 * 1000 * Env.accountRetentionTime);
         } else {
             accountRetentionTime = -1;
         }
@@ -297,7 +292,7 @@ module.exports = function (Env, cb) {
         };
 
         var accountIsActive = function (mtime, pinList, id) {
-            // console.log("id [%s] in premiumSafeKeys", id, premiumSafeKeys.indexOf(id) !== -1); // XXX
+            // console.log("id [%s] in premiumSafeKeys", id, premiumSafeKeys.indexOf(id) !== -1);
             // if their pin log has changed recently then consider them active
             if (mtime && mtime > accountRetentionTime) {
                 return true;
@@ -360,7 +355,7 @@ module.exports = function (Env, cb) {
         };
 
         Pins.load(w(done), {
-            pinPath: config.pinPath,
+            pinPath: Env.paths.pin,
             handler: handler,
         });
     };
@@ -515,9 +510,6 @@ module.exports = function (Env, cb) {
     };
 
     nThen(loadStorage)
-    .nThen(function () {
-        Log.info("EVICT_TIME_TO_LOAD_PINS", msSinceStart());
-    })
     .nThen(removeArchivedChannels)
     .nThen(removeArchivedBlobProofs)
     .nThen(removeArchivedBlobs)

lib/historyKeeper.js

@@ -11,6 +11,7 @@ const Core = require("./commands/core");
 const Store = require("./storage/file");
 const BlobStore = require("./storage/blob");
 const Workers = require("./workers/index");
+//const Eviction = require("./eviction");
 
 module.exports.create = function (config, cb) {
     const Log = config.log;
@@ -26,14 +27,15 @@ module.exports.create = function (config, cb) {
 
     Log.silly('HK_LOADING', 'LOADING HISTORY_KEEPER MODULE');
 
-    // TODO populate Env with everything that you use from config
-    // so that you can stop passing around your raw config
-    // and more easily share state between historyKeeper and rpc
     const Env = {
         Log: Log,
         // store
         id: Crypto.randomBytes(8).toString('hex'),
 
+        inactiveTime: config.inactiveTime,
+        archiveRetentionTime: config.archiveRetentionTime,
+        accountRetentionTime: config.accountRetentionTime,
+
         metadata_cache: {},
         channel_cache: {},
         queueStorage: WriteQueue(),
@@ -254,6 +256,10 @@ module.exports.create = function (config, cb) {
             verbose: config.verbose,
             openFileLimit: config.openFileLimit,
 
+            inactiveTime: config.inactiveTime,
+            archiveRetentionTime: config.archiveRetentionTime,
+            accountRetentionTime: config.accountRetentionTime,
+
             maxWorkers: config.maxWorkers,
         }, w(function (err) {
             if (err) {
@@ -261,8 +267,8 @@ module.exports.create = function (config, cb) {
             }
         }));
     }).nThen(function () {
-        if (config.disableIntegratedTasks) { return; }
         config.intervals = config.intervals || {};
+        if (config.disableIntegratedTasks) { return; }
 
         var tasks_running;
         config.intervals.taskExpiration = setInterval(function () {
@@ -275,6 +281,30 @@ module.exports.create = function (config, cb) {
                 tasks_running = false;
             });
         }, 1000 * 60 * 5); // run every five minutes
+    }).nThen(function () {
+        if (config.disableIntegratedEviction) { return; }
+        const ONE_DAY = 24 * 1000 * 60 * 60;
+        // setting the time of the last eviction to "now"
+        // effectively makes it so that we'll start evicting after the server
+        // has been up for at least one day
+        var last_eviction = +new Date();
+
+        var active = false;
+        config.intervals.eviction = setInterval(function () {
+            if (active) { return; }
+            var now = +new Date();
+            // evict inactive data once per day
+            if (last_eviction && (now - ONE_DAY) < last_eviction) { return; }
+            active = true;
+            Env.evictInactive(function (err) {
+                if (err) {
+                    // NO_INACTIVE_TIME
+                    Log.error('EVICT_INACTIVE_MAIN_ERROR', err);
+                }
+                active = false;
+                last_eviction = now;
+            });
+        }, 60 * 1000);
     }).nThen(function () {
         RPC.create(Env, function (err, _rpc) {
             if (err) { throw err; }

lib/workers/db-worker.js

@@ -13,6 +13,7 @@ const Saferphore = require("saferphore");
 const Logger = require("../log");
 const Tasks = require("../storage/tasks");
 const Nacl = require('tweetnacl/nacl-fast');
+const Eviction = require("../eviction");
 
 const Env = {
     Log: {},
@@ -39,13 +40,21 @@ const init = function (config, _cb) {
         return void cb('E_INVALID_CONFIG');
     }
 
+    Env.paths = {
+        pin: config.pinPath,
+    };
+
+    Env.inactiveTime = config.inactiveTime;
+    Env.archiveRetentionTime = config.archiveRetentionTime;
+    Env.accountRetentionTime = config.accountRetentionTime;
+
     nThen(function (w) {
         Store.create(config, w(function (err, _store) {
             if (err) {
                 w.abort();
                 return void cb(err);
             }
-            store = _store;
+            Env.store = store = _store;
         }));
         Store.create({
             filePath: config.pinPath,
@@ -54,7 +63,7 @@ const init = function (config, _cb) {
                 w.abort();
                 return void cb(err);
             }
-            pinStore = _pinStore;
+            Env.pinStore = pinStore = _pinStore;
         }));
         BlobStore.create({
             blobPath: config.blobPath,
@@ -66,7 +75,7 @@ const init = function (config, _cb) {
                 w.abort();
                 return void cb(err);
             }
-            blobStore = blob;
+            Env.blobStore = blobStore = blob;
         }));
     }).nThen(function (w) {
         Tasks.create({
@@ -436,6 +445,10 @@ const writeTask = function (data, cb) {
     Env.tasks.write(data.time, data.task_command, data.args, cb);
 };
 
+const evictInactive = function (data, cb) {
+    Eviction(Env, cb);
+};
+
 const COMMANDS = {
     COMPUTE_INDEX: computeIndex,
     COMPUTE_METADATA: computeMetadata,
@@ -449,6 +462,7 @@ const COMMANDS = {
     REMOVE_OWNED_BLOB: removeOwnedBlob,
     RUN_TASKS: runTasks,
     WRITE_TASK: writeTask,
+    EVICT_INACTIVE: evictInactive,
 };
 
 COMMANDS.INLINE = function (data, cb) {

lib/workers/index.js

@@ -350,6 +350,12 @@ Workers.initialize = function (Env, config, _cb) {
             }, cb);
         };
 
+        Env.evictInactive = function (cb) {
+            sendCommand({
+                command: 'EVICT_INACTIVE',
+            }, cb);
+        };
+
         Env.runTasks = function (cb) {
             sendCommand({
                 command: 'RUN_TASKS',

scripts/evict-inactive.js

@@ -5,13 +5,17 @@ var BlobStore = require("../lib/storage/blob");
 
 var Quota = require("../lib/commands/quota");
 
+var config = require("../lib/load-config");
 var Env = {
-    config: require("../lib/load-config"),
+    inactiveTime: config.inactiveTime,
+    archiveRetentionTime: config.archiveRetentionTime,
+    accountRetentionTime: config.accountRetentionTime,
+    paths: {
+        pin: config.pinPath,
+    },
 };
 
 var prepareEnv = function (Env, cb) {
-    var config = Env.config;
-
     Env.customLimits = config.customLimits;
     Quota.applyCustomLimits(Env);