From f2ec9cbe33b4a94cfdc780fbc822ae68017bff7b Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 12 Oct 2020 19:18:12 +0530 Subject: [PATCH] load premium and customLimits to avoid evicting them even if inactive --- lib/commands/admin-rpc.js | 1 + lib/commands/quota.js | 19 ++++++++++++----- lib/eviction.js | 33 ++++++++++++++++++++---------- scripts/evict-inactive.js | 43 +++++++++++++++++++++++++++++++-------- 4 files changed, 71 insertions(+), 25 deletions(-) diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index fd2e934f3..a97ffeb2b 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -223,6 +223,7 @@ var instanceStatus = function (Env, Server, cb) { defaultStorageLimit: Env.defaultStorageLimit, lastEviction: Env.lastEviction, + knownActiveAccounts: Env.knownActiveAccounts, }); }; diff --git a/lib/commands/quota.js b/lib/commands/quota.js index bcfedb3f7..b779d8e56 100644 --- a/lib/commands/quota.js +++ b/lib/commands/quota.js @@ -117,13 +117,22 @@ Quota.queryAccountServer = function (Env, cb) { }); }; -Quota.updateCachedLimits = function (Env, cb) { +Quota.shouldContactServer = function (Env) { + return !(Env.blockDailyCheck === true || + ( + typeof(Env.blockDailyCheck) === 'undefined' && + Env.adminEmail === false + && Env.allowSubscriptions === false + ) + ); +}; + +Quota.updateCachedLimits = function (Env, _cb) { + var cb = Util.mkAsync(_cb); + Quota.applyCustomLimits(Env); - if (Env.blockDailyCheck === true || - (typeof(Env.blockDailyCheck) === 'undefined' && Env.adminEmail === false && Env.allowSubscriptions === false)) { - return void cb(); - } + if (!Quota.shouldContactServer(Env)) { return void cb(); } Quota.queryAccountServer(Env, function (err, json) { if (err) { return void cb(err); } if (!json) { return void cb(); } diff --git a/lib/eviction.js b/lib/eviction.js index 1dd511666..979848bc6 100644 --- a/lib/eviction.js +++ b/lib/eviction.js @@ -2,6 +2,7 @@ var nThen = require("nthen"); var Bloom = require("@mcrowe/minibloom"); var Util = require("../lib/common-util"); var Pins = require("../lib/pins"); +var Keys = require("./keys"); var getNewestTime = function (stats) { return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) { @@ -42,12 +43,11 @@ module.exports = function (Env, cb) { // pre-converted to the 'safeKey' format so we can easily compare // them against ids we see on the filesystem var premiumSafeKeys = Object.keys(Env.limits || {}) - .filter(function (key) { - return key.length === 44; + .map(function (id) { + return Keys.canonicalize(id); }) - .map(function (unsafeKey) { - return Util.escapeKeyCharacters(unsafeKey); - }); + .filter(Boolean) + .map(Util.escapeKeyCharacters); // files which have not been changed since before this date can be considered inactive var inactiveTime = +new Date() - (Env.inactiveTime * 24 * 3600 * 1000); @@ -291,7 +291,7 @@ module.exports = function (Env, cb) { return activeDocs.test(docId); }; - var accountIsActive = function (mtime, pinList, id) { + var accountIsActive = function (mtime, pinList) { // console.log("id [%s] in premiumSafeKeys", id, premiumSafeKeys.indexOf(id) !== -1); // if their pin log has changed recently then consider them active if (mtime && mtime > accountRetentionTime) { @@ -299,11 +299,10 @@ module.exports = function (Env, cb) { } // iterate over their pinned documents until you find one that has been active - if (pinList.some(docIsActive)) { - return true; - } + return pinList.some(docIsActive); + }; - // Finally, make sure it's not a premium account + var isPremiumAccount = function (id) { return premiumSafeKeys.indexOf(id) !== -1; }; @@ -317,7 +316,7 @@ module.exports = function (Env, cb) { var mtime = content.latest; var pinList = Object.keys(content.pins); - if (accountIsActive(mtime, pinList, id)) { + if (accountIsActive(mtime, pinList)) { // add active accounts' pinned documents to a second bloom filter pinAll(pinList); return void next(); @@ -332,6 +331,15 @@ module.exports = function (Env, cb) { return void next(); } + if (isPremiumAccount(id)) { + Log.info("EVICT_INACTIVE_PREMIUM_ACCOUNT", { + id: id, + mtime: mtime, + }); + pinAll(pinList); + return void next(); + } + // remove the pin logs of inactive accounts if inactive account removal is configured pinStore.archiveChannel(id, function (err) { if (err) { @@ -348,6 +356,9 @@ module.exports = function (Env, cb) { "EVICT_COUNT_ACCOUNTS": "EVICT_INACTIVE_ACCOUNTS"; + // update the number of known active accounts in Env for statistics + Env.knownActiveAccounts = accounts - inactive; + Log.info(label, { accounts: accounts, inactive: inactive, diff --git a/scripts/evict-inactive.js b/scripts/evict-inactive.js index 91b7dad63..bf7e1ca5b 100644 --- a/scripts/evict-inactive.js +++ b/scripts/evict-inactive.js @@ -4,20 +4,41 @@ var Store = require("../lib/storage/file"); var BlobStore = require("../lib/storage/blob"); var Quota = require("../lib/commands/quota"); +var Environment = require("../lib/env"); +var Decrees = require("../lib/decrees"); var config = require("../lib/load-config"); -var Env = { - inactiveTime: config.inactiveTime, - archiveRetentionTime: config.archiveRetentionTime, - accountRetentionTime: config.accountRetentionTime, - paths: { - pin: config.pinPath, - }, + +var Env = Environment.create(config); + +var loadPremiumAccounts = function (Env, cb) { + nThen(function (w) { + // load premium accounts + Quota.updateCachedLimits(Env, w(function (err) { + if (err) { + Env.Log.error('EVICT_LOAD_PREMIUM_ACCOUNTS', { + error: err, + }); + } + })); + }).nThen(function (w) { + // load and apply decrees + Decrees.load(Env, w(function (err) { + if (err) { + Env.Log.error('EVICT_LOAD_DECREES', { + error: err.code || err, + message: err.message, + }); + } + })); + }).nThen(function () { + //console.log(Env.limits); + cb(); + }); }; var prepareEnv = function (Env, cb) { - Env.customLimits = config.customLimits; - Quota.applyCustomLimits(Env); + //Quota.applyCustomLimits(Env); nThen(function (w) { /* Database adaptors @@ -58,6 +79,10 @@ var prepareEnv = function (Env, cb) { } Env.blobStore = _; })); + }).nThen(function (w) { + loadPremiumAccounts(Env, w(function (/* err */) { + //if (err) { } + })); }).nThen(function () { cb(); });