From b13f56247f2b2ebee370058611eab462e19b274c Mon Sep 17 00:00:00 2001 From: yflory Date: Thu, 12 Mar 2020 15:53:59 +0100 Subject: [PATCH 1/8] Fix trim history --- www/common/common-ui-elements.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index f407826e9..eeceda235 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -146,6 +146,17 @@ define([ if (data.href) { data.href = base + data.href; } if (data.roHref) { data.roHref = base + data.roHref; } }), opts.href); + + // If this is a file, don't try to look for metadata + if (opts.channel && opts.channel.length > 34) { return; } + common.getPadMetadata({ + channel: opts.channel // optional, fallback to current pad + }, waitFor(function (obj) { + if (obj && obj.error) { return; } + data.owners = obj.owners; + data.expire = obj.expire; + data.pending_owners = obj.pending_owners; + })); }).nThen(function () { cb(void 0, data); }); From bc034d95a3ade10252b151a3ecfd525d75fcba2d Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 12 Mar 2020 11:13:23 -0400 Subject: [PATCH 2/8] enforce asynchrony in 'gethistoryOffset' --- lib/hk-util.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/hk-util.js b/lib/hk-util.js index 3f0aed360..cb9e9b8ef 100644 --- a/lib/hk-util.js +++ b/lib/hk-util.js @@ -436,7 +436,8 @@ const storeMessage = function (Env, channel, msg, isCp, optionalMessageHash) { * -1 if you didn't find it */ -const getHistoryOffset = (Env, channelName, lastKnownHash, cb) => { +const getHistoryOffset = (Env, channelName, lastKnownHash, _cb) => { + const cb = Util.once(Util.mkAsync(_cb)); const store = Env.store; const Log = Env.Log; From e1a0daac9c02da81743c0cf2100625c9b3b2c86c Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 12 Mar 2020 11:23:43 -0400 Subject: [PATCH 3/8] possible server fixes --- lib/storage/file.js | 5 ++++- lib/stream-file.js | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/storage/file.js b/lib/storage/file.js index 6d2c672a5..053ecc9c7 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -101,6 +101,9 @@ var getMetadataAtPath = function (Env, path, _cb) { if (i++ > 0) { console.log("aborting"); abort(); + try { stream.close(); } catch (err) { + console.log("could not close stream"); + } return void cb(); } var metadata; @@ -890,7 +893,7 @@ var trimChannel = function (env, channelName, hash, _cb) { var retain = false; var handler = function (msgObj, readMore, abort) { - if (ABORT) { return void abort(); } + if (ABORT) { return void abort(); } // XXX // the first message might be metadata... ignore it if so if (i++ === 0 && msgObj.buff.indexOf('{') === 0) { return readMore(); diff --git a/lib/stream-file.js b/lib/stream-file.js index dc44aaf50..c3130365b 100644 --- a/lib/stream-file.js +++ b/lib/stream-file.js @@ -67,7 +67,15 @@ Stream.readFileBin = (stream, msgHandler, cb) => { mkBufferSplit(), mkOffsetCounter(), Pull.asyncMap((data, moreCb) => { - msgHandler(data, moreCb, () => { keepReading = false; moreCb(); }); + msgHandler(data, moreCb, () => { + try { + stream.close(); + } catch (err) { + console.error("READ_FILE_BIN_ERR", err); + } + keepReading = false; + moreCb(); + }); }), Pull.drain(() => (keepReading), (err) => { cb((keepReading) ? err : undefined); From 12300579701f65eb0ccc3ad33049435690dfdf6a Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 12 Mar 2020 11:31:08 -0400 Subject: [PATCH 4/8] disable IS_PAD_PINNED --- lib/commands/pin-rpc.js | 1 + lib/rpc.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/commands/pin-rpc.js b/lib/commands/pin-rpc.js index 2888f1e61..11648fced 100644 --- a/lib/commands/pin-rpc.js +++ b/lib/commands/pin-rpc.js @@ -546,6 +546,7 @@ const deferResponse = function (Env, channel, cb) { }; Pinning.isChannelPinned = function (Env, channel, cb) { + return void cb(void 0, true); // XXX // if the pins are fully loaded then you can answer yes/no definitively if (Env.pinsLoaded) { return void cb(void 0, !isEmpty(Env.pinnedPads[channel])); diff --git a/lib/rpc.js b/lib/rpc.js index c54edf31d..241f77d41 100644 --- a/lib/rpc.js +++ b/lib/rpc.js @@ -198,7 +198,7 @@ RPC.create = function (Env, cb) { updateLimitDaily(); Env.intervals.dailyLimitUpdate = setInterval(updateLimitDaily, 24*3600*1000); - Pinning.loadChannelPins(Env); + //Pinning.loadChannelPins(Env); // XXX // expire old sessions once per minute Env.intervals.sessionExpirationInterval = setInterval(function () { From f644c24362efcb39e6d65d00a08761662054c7a7 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 12 Mar 2020 11:35:16 -0400 Subject: [PATCH 5/8] add an admin-rpc to count the number of open files --- lib/commands/admin-rpc.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 13d6a8c36..492d06d7e 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -7,6 +7,13 @@ var Fs = require("fs"); var Admin = module.exports; +var getFileDescriptorCount = function (Env, server, cb) { + Fs.readdir('/proc/self/fd', function(err, list) { + if (err) { return void cb(err); } + cb(void 0, list.length); + }); +}; + var getActiveSessions = function (Env, Server, cb) { var stats = Server.getSessionStats(); cb(void 0, [ @@ -105,6 +112,7 @@ var commands = { DISK_USAGE: getDiskUsage, FLUSH_CACHE: flushCache, SHUTDOWN: shutdown, + GET_FILE_DESCRIPTOR_COUNT: getFileDescriptorCount, }; Admin.command = function (Env, safeKey, data, _cb, Server) { From 81c7416095b088cae83776fd526bdcb16f6d7dc3 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 12 Mar 2020 11:40:46 -0400 Subject: [PATCH 6/8] lint compliance --- lib/commands/pin-rpc.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/commands/pin-rpc.js b/lib/commands/pin-rpc.js index 11648fced..c15d64e30 100644 --- a/lib/commands/pin-rpc.js +++ b/lib/commands/pin-rpc.js @@ -547,6 +547,7 @@ const deferResponse = function (Env, channel, cb) { Pinning.isChannelPinned = function (Env, channel, cb) { return void cb(void 0, true); // XXX +/* // if the pins are fully loaded then you can answer yes/no definitively if (Env.pinsLoaded) { return void cb(void 0, !isEmpty(Env.pinnedPads[channel])); @@ -560,5 +561,6 @@ Pinning.isChannelPinned = function (Env, channel, cb) { // as you learn that one account has pinned a file. // negative responses have to wait until the end deferResponse(Env, channel, cb); +*/ }; From 2d47e7e2cfcdcf287774b814e66e625b4ad5c0dd Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 12 Mar 2020 12:09:44 -0400 Subject: [PATCH 7/8] implement an admin rpc to fetch ulimit data --- lib/commands/admin-rpc.js | 6 ++++++ package-lock.json | 5 +++++ package.json | 1 + 3 files changed, 12 insertions(+) diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index 492d06d7e..7b4334c24 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -2,6 +2,7 @@ const nThen = require("nthen"); const getFolderSize = require("get-folder-size"); const Util = require("../common-util"); +const Ulimit = require("ulimit"); var Fs = require("fs"); @@ -14,6 +15,10 @@ var getFileDescriptorCount = function (Env, server, cb) { }); }; +var getFileDescriptorLimit = function (env, server, cb) { + Ulimit(cb); +}; + var getActiveSessions = function (Env, Server, cb) { var stats = Server.getSessionStats(); cb(void 0, [ @@ -113,6 +118,7 @@ var commands = { FLUSH_CACHE: flushCache, SHUTDOWN: shutdown, GET_FILE_DESCRIPTOR_COUNT: getFileDescriptorCount, + GET_FILE_DESCRIPTOR_LIMIT: getFileDescriptorLimit, }; Admin.command = function (Env, safeKey, data, _cb, Server) { diff --git a/package-lock.json b/package-lock.json index 6cafde975..f86eb6eb7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1244,6 +1244,11 @@ "mime-types": "~2.1.24" } }, + "ulimit": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/ulimit/-/ulimit-0.0.2.tgz", + "integrity": "sha1-K1H53IOBrkECY2zsXrM4wmMFiKA=" + }, "ultron": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz", diff --git a/package.json b/package.json index 51bfec3f7..c3999f893 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "sortify": "^1.0.4", "stream-to-pull-stream": "^1.7.2", "tweetnacl": "~0.12.2", + "ulimit": "0.0.2", "ws": "^3.3.1" }, "devDependencies": { From b5560c279b1c73c42f1564e1df83933db4faef2c Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 13 Mar 2020 13:43:06 -0400 Subject: [PATCH 8/8] label some issues to fix before the release --- www/kanban/inner.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/www/kanban/inner.js b/www/kanban/inner.js index 6aca4ac67..185a768dd 100644 --- a/www/kanban/inner.js +++ b/www/kanban/inner.js @@ -102,6 +102,8 @@ define([ } html += cursor.name + ''; + // XXX breaks on "😶" + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/charAt#Getting_whole_characters var l = (cursor.name || Messages.anonymous).slice(0,1).toUpperCase(); var text = ''; @@ -777,6 +779,7 @@ define([ var boardExists = function (b) { return b.id === "board" + counter; }; while (kanban.options.boards.some(boardExists)) { counter++; } */ + // XXX confirm that it doesn't collide with an existing id? unlikely but possible var id = Util.createRandomInteger(); kanban.addBoard({