From 1f2e45d6c8eb948f5225f58d60ccbdc2058ac129 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 12 Oct 2016 10:52:27 +0200 Subject: [PATCH] improved fs storage adaptor and config docs * regularly close open file descriptors older than channelExpirationMs * clean up older file descriptors when exceeding openFileLimit --- config.js.dist | 46 +++++++++++++++++++++---------- storage/file.js | 73 +++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 99 insertions(+), 20 deletions(-) diff --git a/config.js.dist b/config.js.dist index c75748508..5f942bbfb 100644 --- a/config.js.dist +++ b/config.js.dist @@ -11,18 +11,18 @@ module.exports = { httpPort: 3000, /* your server's websocket url is configurable - (default: '/cryptpad_websocket') - - websocketPath can be relative, of the form '/path/to/websocket' - or absolute, specifying a particular URL - - 'wss://cryptpad.fr:3000/cryptpad_websocket' - */ + * (default: '/cryptpad_websocket') + * + * websocketPath can be relative, of the form '/path/to/websocket' + * or absolute, specifying a particular URL + * + * 'wss://cryptpad.fr:3000/cryptpad_websocket' + */ websocketPath: '/cryptpad_websocket', /* it is assumed that your websocket will bind to the same port as http - you can override this behaviour by supplying a number via websocketPort - */ + * you can override this behaviour by supplying a number via websocketPort + */ //websocketPort: 3000, @@ -31,12 +31,11 @@ module.exports = { */ logToStdout: false, - /* - Cryptpad stores each document in an individual file on your hard drive. - Specify a directory where files should be stored. - It will be created automatically if it does not already exist. - */ - filePath: './datastore/', + /* Cryptpad supports verbose logging + * (false by default) + */ + verbose: false, + /* You have the option of specifying an alternative storage adaptor. @@ -56,6 +55,23 @@ module.exports = { */ storage: './storage/file', + /* + Cryptpad stores each document in an individual file on your hard drive. + Specify a directory where files should be stored. + It will be created automatically if it does not already exist. + */ + filePath: './datastore/', + + /* Cryptpad's file storage adaptor closes unused files after a configurale + * number of milliseconds (default 30000 (30 seconds)) + */ + channelExpirationMs: 30000, + + /* Cryptpad's file storage adaptor is limited by the number of open files. + * When the adaptor reaches openFileLimit, it will clean up older files + */ + openFileLimit: 2048, + /* it is recommended that you serve cryptpad over https * the filepaths below are used to configure your certificates */ diff --git a/storage/file.js b/storage/file.js index 162894b63..1584f4a59 100644 --- a/storage/file.js +++ b/storage/file.js @@ -47,9 +47,49 @@ var checkPath = function (path, callback) { }); }; +var removeChannel = function (env, channelName, cb) { + var filename = Path.join(env.root, channelName.slice(0, 2), channelName + '.ndjson'); + Fs.unlink(filename, cb); +}; + +var closeChannel = function (env, channelName, cb) { + if (!env.channels[channelName]) { return; } + try { + env.channels[channelName].writeStream.close(); + delete env.channels[channelName]; + env.openFiles--; + cb(); + } catch (err) { + cb(err); + } +}; + +var flushUnusedChannels = function (env, cb, frame) { + var currentTime = +new Date(); + + var expiration = typeof(frame) === 'undefined'? env.channelExpirationMs: frame; + Object.keys(env.channels).forEach(function (chanId) { + var chan = env.channels[chanId]; + if (typeof(chan.atime) !== 'number') { return; } + if (currentTime >= expiration + chan.atime) { + closeChannel(env, chanId, function (err) { + if (err) { + console.error(err); + return; + } + if (env.verbose) { + console.log("Closed channel [%s]", chanId); + } + }); + } + }); + cb(); +}; + var getChannel = function (env, id, callback) { if (env.channels[id]) { var chan = env.channels[id]; + chan.atime = +new Date(); if (chan.whenLoaded) { chan.whenLoaded.push(callback); } else { @@ -57,6 +97,19 @@ var getChannel = function (env, id, callback) { } return; } + + if (env.openFiles >= env.openFileLimit) { + // if you're running out of open files, asynchronously clean up expired files + // do it on a shorter timeframe, though (half of normal) + setTimeout(function () { + flushUnusedChannels(env, function () { + if (env.verbose) { + console.log("Approaching open file descriptor limit. Cleaning up"); + } + }, env.channelExpirationMs / 2); + }); + } + var channel = env.channels[id] = { atime: +new Date(), messages: [], @@ -100,8 +153,10 @@ var getChannel = function (env, id, callback) { }).nThen(function (waitFor) { if (errorState) { return; } var stream = channel.writeStream = Fs.createWriteStream(path, { flags: 'a' }); + env.openFiles++; stream.on('open', waitFor()); stream.on('error', function (err) { + env.openFiles--; // this might be called after this nThen block closes. if (channel.whenLoaded) { complete(err); @@ -161,15 +216,14 @@ var getMessages = function (env, chanName, handler, cb) { }); }; -var removeChannel = function (env, channelName, cb) { - var filename = Path.join(env.root, channelName.slice(0, 2), channelName + '.ndjson'); - Fs.unlink(filename, cb); -}; - module.exports.create = function (conf, cb) { var env = { root: conf.filePath || './datastore', channels: { }, + channelExpirationMs: conf.channelExpirationMs || 30000, + verbose: conf.verbose, + openFiles: 0, + openFileLimit: conf.openFileLimit || 2048, }; Fs.mkdir(env.root, function (err) { if (err && err.code !== 'EEXIST') { @@ -188,6 +242,15 @@ module.exports.create = function (conf, cb) { cb(err); }); }, + closeChannel: function (channelName, cb) { + closeChannel(env, channelName, cb); + }, + flushUnusedChannels: function (cb) { + flushUnusedChannels(env, cb); + }, }); }); + setInterval(function () { + flushUnusedChannels(env, function () { }); + }, 5000); };