diff --git a/scripts/evict-inactive.js b/scripts/evict-inactive.js index f23cbd7cd..d1026fd64 100644 --- a/scripts/evict-inactive.js +++ b/scripts/evict-inactive.js @@ -51,97 +51,115 @@ nThen(function (w) { // count the number of files which have been removed in this run var removed = 0; - var handler = function (err, item) { - if (err) { return void Log.error('EVICT_ARCHIVED_CHANNEL_ITERATION', err); } + var handler = function (err, item, cb) { + if (err) { + Log.error('EVICT_ARCHIVED_CHANNEL_ITERATION', err); + return void cb(); + } // don't mess with files that are freshly stored in cold storage // based on ctime because that's changed when the file is moved... - if (+new Date(item.ctime) > retentionTime) { return; } + if (+new Date(item.ctime) > retentionTime) { + return void cb(); + } // but if it's been stored for the configured time... // expire it store.removeArchivedChannel(item.channel, w(function (err) { if (err) { - return void Log.error('EVICT_ARCHIVED_CHANNEL_REMOVAL_ERROR', { + Log.error('EVICT_ARCHIVED_CHANNEL_REMOVAL_ERROR', { error: err, channel: item.channel, }); + return void cb(); } Log.info('EVICT_ARCHIVED_CHANNEL_REMOVAL', item.channel); removed++; + cb(); })); }; // if you hit an error, log it // otherwise, when there are no more channels to process // log some stats about how many were removed - var cb = function (err) { + var done = function (err) { if (err) { return Log.error('EVICT_ARCHIVED_FINAL_ERROR', err); } Log.info('EVICT_ARCHIVED_CHANNELS_REMOVED', removed); }; - store.listArchivedChannels(handler, w(cb)); + store.listArchivedChannels(handler, w(done)); }).nThen(function (w) { var removed = 0; var channels = 0; var archived = 0; - var handler = function (err, item) { + var handler = function (err, item, cb) { channels++; - if (err) { return void Log.error('EVICT_CHANNEL_ITERATION', err); } + if (err) { + Log.error('EVICT_CHANNEL_ITERATION', err); + return void cb(); + } // check if the database has any ephemeral channels // if it does it's because of a bug, and they should be removed if (item.channel.length === 34) { return void store.removeChannel(item.channel, w(function (err) { if (err) { - return void Log.error('EVICT_EPHEMERAL_CHANNEL_REMOVAL_ERROR', { + Log.error('EVICT_EPHEMERAL_CHANNEL_REMOVAL_ERROR', { error: err, channel: item.channel, }); + return void cb(); } Log.info('EVICT_EPHEMERAL_CHANNEL_REMOVAL', item.channel); })); } // bail out if the channel was modified recently - if (+new Date(item.mtime) > inactiveTime) { return; } + if (+new Date(item.mtime) > inactiveTime) { return void cb(); } // ignore the channel if it's pinned - if (pins[item.channel]) { return; } + if (pins[item.channel]) { return void cb(); } // if the server is configured to retain data, archive the channel if (config.retainData) { - store.archiveChannel(item.channel, w(function (err) { - if (err) { return void Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', { - error: err, - channel: item.channel, - }); } + return void store.archiveChannel(item.channel, w(function (err) { + if (err) { + Log.error('EVICT_CHANNEL_ARCHIVAL_ERROR', { + error: err, + channel: item.channel, + }); + return void cb(); + } Log.info('EVICT_CHANNEL_ARCHIVAL', item.channel); archived++; + cb(); })); - return; } // otherwise remove it store.removeChannel(item.channel, w(function (err) { - if (err) { return void Log.error('EVICT_CHANNEL_REMOVAL_ERROR', { - error: err, - channel: item.channel, - }); } + if (err) { + Log.error('EVICT_CHANNEL_REMOVAL_ERROR', { + error: err, + channel: item.channel, + }); + return void cb(); + } Log.info('EVICT_CHANNEL_REMOVAL', item.channel); removed++; + cb(); })); }; - var cb = function () { + var done = function () { if (config.retainData) { return void Log.info('EVICT_CHANNELS_ARCHIVED', archived); } return void Log.info('EVICT_CHANNELS_REMOVED', removed); }; - store.listChannels(handler, w(cb)); + store.listChannels(handler, w(done)); }).nThen(function () { // the store will keep this script running if you don't shut it down store.shutdown(); diff --git a/scripts/restored-archived.js b/scripts/restored-archived.js new file mode 100644 index 000000000..4cefc764b --- /dev/null +++ b/scripts/restored-archived.js @@ -0,0 +1,75 @@ +var nThen = require("nthen"); + +var Store = require("../storage/file"); +var Pinned = require("./pinned"); +var config = require("../lib/load-config"); + +var store; +var pins; +var Log; +nThen(function (w) { + // load the store which will be used for iterating over channels + // and performing operations like archival and deletion + Store.create(config, w(function (_) { + store = _; + })); // load the list of pinned files so you know which files + // should not be archived or deleted + Pinned.load(w(function (err, _) { + if (err) { + w.abort(); + return void console.error(err); + } + pins = _; + }), { + pinPath: config.pinPath, + }); + + // load the logging module so that you have a record of which + // files were archived or deleted at what time + var Logger = require("../lib/log"); + Logger.create(config, w(function (_) { + Log = _; + })); +}).nThen(function (w) { + // count the number of files which have been restored in this run + var restored = 0; + + var handler = function (err, item, cb) { + if (err) { + Log.error('RESTORE_ARCHIVED_CHANNEL_ITERATION', err); + return void cb(); + } + + // but if it's been stored for the configured time... + // expire it + store.removeArchivedChannel(item.channel, w(function (err) { + if (err) { + Log.error('RESTORE_ARCHIVED_CHANNEL_RESTORATION_ERROR', { + error: err, + channel: item.channel, + }); + return void cb(); + } + Log.info('RESTORE_ARCHIVED_CHANNEL_RESTORATION', item.channel); + restored++; + cb(); + })); + }; + + // if you hit an error, log it + // otherwise, when there are no more channels to process + // log some stats about how many were removed + var done = function (err) { + if (err) { + return Log.error('RESTORE_ARCHIVED_FINAL_ERROR', err); + } + Log.info('RESTORE_ARCHIVED_CHANNELS_RESTORED', restored); + }; + + store.listArchivedChannels(handler, w(done)); +}).nThen(function () { + // the store will keep this script running if you don't shut it down + store.shutdown(); + Log.shutdown(); +}); + diff --git a/storage/file.js b/storage/file.js index 1f8c00a0c..f92600c46 100644 --- a/storage/file.js +++ b/storage/file.js @@ -253,7 +253,8 @@ var listChannels = function (root, handler, cb) { // otherwise throw it on the pile sema.take(function (give) { - Fs.stat(filepath, w(give(function (err, stats) { + var next = give(); + Fs.stat(filepath, w(function (err, stats) { if (err) { return void handler(err); } @@ -264,8 +265,8 @@ var listChannels = function (root, handler, cb) { mtime: stats.mtime, ctime: stats.ctime, size: stats.size, - }); - }))); + }, next); + })); }); }); })));