infra
/
cryptpad
2
1
Fork 0
Code Issues Pull Requests Projects Releases 1 Activity

batch concurrent requests for expensive IO and computations

Browse Source
pull/1/head
ansuz 5 years ago
parent ed82936610
commit 65b0112dcb
1 changed files with 106 additions and 88 deletions
Show Stats Download Patch File Download Diff File

194
rpc.js
Unescape Escape View File

@ -19,6 +19,7 @@ const getFolderSize = require("get-folder-size");
const Pins = require("./lib/pins"); const Pins = require("./lib/pins");
const Meta = require("./lib/metadata"); const Meta = require("./lib/metadata");
const WriteQueue = require("./lib/write-queue"); const WriteQueue = require("./lib/write-queue");
const BatchRead = require("./lib/batch-read");
var RPC = module.exports; var RPC = module.exports;
@ -231,6 +232,7 @@ var checkSignature = function (signedMsg, signature, publicKey) {
return Nacl.sign.detached.verify(signedBuffer, signatureBuffer, pubBuffer); return Nacl.sign.detached.verify(signedBuffer, signatureBuffer, pubBuffer);
}; };
const batchUserPins = BatchRead();
var loadUserPins = function (Env, publicKey, cb) { var loadUserPins = function (Env, publicKey, cb) {
var session = getSession(Env.Sessions, publicKey); var session = getSession(Env.Sessions, publicKey);
@ -238,21 +240,23 @@ var loadUserPins = function (Env, publicKey, cb) {
return cb(session.channels); return cb(session.channels);
} }
var ref = {}; batchUserPins(publicKey, cb, function (done) {
var lineHandler = Pins.createLineHandler(ref, function (label, data) { var ref = {};
Log.error(label, { var lineHandler = Pins.createLineHandler(ref, function (label, data) {
log: publicKey, Log.error(label, {
data: data, log: publicKey,
data: data,
});
}); });
});
// if channels aren't in memory. load them from disk // if channels aren't in memory. load them from disk
Env.pinStore.getMessages(publicKey, lineHandler, function () { Env.pinStore.getMessages(publicKey, lineHandler, function () {
// no more messages // no more messages
// only put this into the cache if it completes // only put this into the cache if it completes
session.channels = ref.pins; session.channels = ref.pins;
cb(ref.pins); done(ref.pins); // FIXME no error handling?
});
}); });
}; };
@ -290,45 +294,49 @@ var getUploadSize = function (Env, channel, cb) {
}); });
}; };
const batchFileSize = BatchRead();
var getFileSize = function (Env, channel, cb) { var getFileSize = function (Env, channel, cb) {
if (!isValidId(channel)) { return void cb('INVALID_CHAN'); } if (!isValidId(channel)) { return void cb('INVALID_CHAN'); }
batchFileSize(channel, cb, function (done) {
if (channel.length === 32) {
if (typeof(Env.msgStore.getChannelSize) !== 'function') {
return done('GET_CHANNEL_SIZE_UNSUPPORTED');
}
if (channel.length === 32) { return void Env.msgStore.getChannelSize(channel, function (e, size /*:number*/) {
if (typeof(Env.msgStore.getChannelSize) !== 'function') { if (e) {
return cb('GET_CHANNEL_SIZE_UNSUPPORTED'); if (e.code === 'ENOENT') { return void done(void 0, 0); }
return void done(e.code);
}
done(void 0, size);
});
} }
return void Env.msgStore.getChannelSize(channel, function (e, size /*:number*/) { // 'channel' refers to a file, so you need another API
if (e) { getUploadSize(Env, channel, function (e, size) {
if (e.code === 'ENOENT') { return void cb(void 0, 0); } if (typeof(size) === 'undefined') { return void done(e); }
return void cb(e.code); done(void 0, size);
}
cb(void 0, size);
}); });
}
// 'channel' refers to a file, so you need another API
getUploadSize(Env, channel, function (e, size) {
if (typeof(size) === 'undefined') { return void cb(e); }
cb(void 0, size);
}); });
}; };
const batchMetadata = BatchRead();
var getMetadata = function (Env, channel, cb) { var getMetadata = function (Env, channel, cb) {
if (!isValidId(channel)) { return void cb('INVALID_CHAN'); } if (!isValidId(channel)) { return void cb('INVALID_CHAN'); }
if (channel.length !== 32) { return cb("INVALID_CHAN"); } if (channel.length !== 32) { return cb("INVALID_CHAN"); }
var ref = {}; batchMetadata(channel, cb, function (done) {
var lineHandler = Meta.createLineHandler(ref, Log.error); var ref = {};
var lineHandler = Meta.createLineHandler(ref, Log.error);
return void Env.msgStore.readChannelMetadata(channel, lineHandler, function (err) { return void Env.msgStore.readChannelMetadata(channel, lineHandler, function (err) {
if (err) { if (err) {
// stream errors? // stream errors?
return void cb(err); return void done(err);
} }
cb(void 0, ref.meta); done(void 0, ref.meta);
});
}); });
}; };
@ -470,19 +478,22 @@ var getDeletedPads = function (Env, channels, cb) {
}); });
}; };
const batchTotalSize = BatchRead();
var getTotalSize = function (Env, publicKey, cb) { var getTotalSize = function (Env, publicKey, cb) {
var bytes = 0; batchTotalSize(publicKey, cb, function (done) {
return void getChannelList(Env, publicKey, function (channels) { var bytes = 0;
if (!channels) { return cb('INVALID_PIN_LIST'); } // unexpected return void getChannelList(Env, publicKey, function (channels) {
if (!channels) { return done('INVALID_PIN_LIST'); } // unexpected
var count = channels.length;
if (!count) { cb(void 0, 0); } var count = channels.length;
if (!count) { return void done(void 0, 0); }
channels.forEach(function (channel) {
getFileSize(Env, channel, function (e, size) { channels.forEach(function (channel) { // FIXME this might as well be nThen
count--; getFileSize(Env, channel, function (e, size) {
if (!e) { bytes += size; } count--;
if (count === 0) { return cb(void 0, bytes); } if (!e) { bytes += size; }
if (count === 0) { return done(void 0, bytes); }
});
}); });
}); });
}); });
@ -1605,53 +1616,60 @@ var writePrivateMessage = function (Env, args, nfwssCtx, cb) {
}); });
}; };
const batchDiskUsage = BatchRead();
var getDiskUsage = function (Env, cb) { var getDiskUsage = function (Env, cb) {
var data = {}; batchDiskUsage('', cb, function (done) {
nThen(function (waitFor) { var data = {};
getFolderSize('./', waitFor(function(err, info) { nThen(function (waitFor) {
data.total = info; getFolderSize('./', waitFor(function(err, info) {
})); data.total = info;
getFolderSize(Env.paths.pin, waitFor(function(err, info) { }));
data.pin = info; getFolderSize(Env.paths.pin, waitFor(function(err, info) {
})); data.pin = info;
getFolderSize(Env.paths.blob, waitFor(function(err, info) { }));
data.blob = info; getFolderSize(Env.paths.blob, waitFor(function(err, info) {
})); data.blob = info;
getFolderSize(Env.paths.staging, waitFor(function(err, info) { }));
data.blobstage = info; getFolderSize(Env.paths.staging, waitFor(function(err, info) {
})); data.blobstage = info;
getFolderSize(Env.paths.block, waitFor(function(err, info) { }));
data.block = info; getFolderSize(Env.paths.block, waitFor(function(err, info) {
})); data.block = info;
getFolderSize(Env.paths.data, waitFor(function(err, info) { }));
data.datastore = info; getFolderSize(Env.paths.data, waitFor(function(err, info) {
})); data.datastore = info;
}).nThen(function () { }));
cb (void 0, data); }).nThen(function () {
done(void 0, data);
});
}); });
}; };
const batchRegisteredUsers = BatchRead();
var getRegisteredUsers = function (Env, cb) { var getRegisteredUsers = function (Env, cb) {
var dir = Env.paths.pin; batchRegisteredUsers('', cb, function (done) {
var folders; var dir = Env.paths.pin;
var users = 0; var folders;
nThen(function (waitFor) { var users = 0;
Fs.readdir(dir, waitFor(function (err, list) { nThen(function (waitFor) {
if (err) {
waitFor.abort();
return void cb(err);
}
folders = list;
}));
}).nThen(function (waitFor) {
folders.forEach(function (f) {
var dir = Env.paths.pin + '/' + f;
Fs.readdir(dir, waitFor(function (err, list) { Fs.readdir(dir, waitFor(function (err, list) {
if (err) { return; } if (err) {
users += list.length; waitFor.abort();
return void done(err);
}
folders = list;
})); }));
}).nThen(function (waitFor) {
folders.forEach(function (f) {
var dir = Env.paths.pin + '/' + f;
Fs.readdir(dir, waitFor(function (err, list) {
if (err) { return; }
users += list.length;
}));
});
}).nThen(function () {
done(void 0, users);
}); });
}).nThen(function () {
cb(void 0, users);
}); });
}; };
var getActiveSessions = function (Env, ctx, cb) { var getActiveSessions = function (Env, ctx, cb) {

Write Preview
Loading…
Cancel
Save