introduce some artificial delays in the eviction script

...and reduce the margin of error in its bloom filters
pull/1/head
ansuz 4 years ago
parent 90f046f896
commit a29c5641b8

@ -81,7 +81,9 @@ module.exports = function (Env, cb) {
TODO make this configurable ? TODO make this configurable ?
*/ */
var BLOOM_CAPACITY = (1 << 20) - 1; // over a million items var BLOOM_CAPACITY = (1 << 20) - 1; // over a million items
var BLOOM_ERROR = 1 / 1000; // an error rate of one in a thousand var BLOOM_ERROR = 1 / 10000; // an error rate of one in a thousand
// the number of ms artificially introduced between CPU-intensive operations
var THROTTLE_FACTOR = 10;
// we'll use one filter for the set of active documents // we'll use one filter for the set of active documents
var activeDocs = Bloom.optimalFilter(BLOOM_CAPACITY, BLOOM_ERROR); var activeDocs = Bloom.optimalFilter(BLOOM_CAPACITY, BLOOM_ERROR);
@ -165,6 +167,7 @@ module.exports = function (Env, cb) {
// if they are older than the specified retention time // if they are older than the specified retention time
var removed = 0; var removed = 0;
blobs.list.archived.proofs(function (err, item, next) { blobs.list.archived.proofs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
if (err) { if (err) {
Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err); Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err);
return void next(); return void next();
@ -190,6 +193,7 @@ module.exports = function (Env, cb) {
// if they are older than the specified retention time // if they are older than the specified retention time
var removed = 0; var removed = 0;
blobs.list.archived.blobs(function (err, item, next) { blobs.list.archived.blobs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
if (err) { if (err) {
Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err); Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err);
return void next(); return void next();
@ -246,6 +250,7 @@ module.exports = function (Env, cb) {
var active = 0; var active = 0;
blobs.list.blobs(function (err, item, next) { blobs.list.blobs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
n_blobs++; n_blobs++;
if (err) { if (err) {
Log.error("EVICT_BLOB_CATEGORIZATION", err); Log.error("EVICT_BLOB_CATEGORIZATION", err);
@ -311,6 +316,7 @@ module.exports = function (Env, cb) {
// otherwise, we'll only retain data from active accounts // otherwise, we'll only retain data from active accounts
// so we need more heuristics // so we need more heuristics
var handler = function (content, id, next) { var handler = function (content, id, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
accounts++; accounts++;
var mtime = content.latest; var mtime = content.latest;
@ -416,6 +422,7 @@ module.exports = function (Env, cb) {
// if they don't correspond to a pinned or active file // if they don't correspond to a pinned or active file
var removed = 0; var removed = 0;
blobs.list.proofs(function (err, item, next) { blobs.list.proofs(function (err, item, next) {
next = Util.mkAsync(next, THROTTLE_FACTOR);
if (err) { if (err) {
next(); next();
return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err); return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err);
@ -458,6 +465,7 @@ module.exports = function (Env, cb) {
var archived = 0; var archived = 0;
var handler = function (err, item, cb) { var handler = function (err, item, cb) {
cb = Util.mkAsync(cb, THROTTLE_FACTOR);
channels++; channels++;
if (err) { if (err) {
Log.error('EVICT_CHANNEL_ITERATION', err); Log.error('EVICT_CHANNEL_ITERATION', err);

Loading…
Cancel
Save