From 46cd796db552c66a7450e3c1d64e4fc4e6533efe Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 25 Jun 2019 13:43:15 +0200 Subject: [PATCH 001/258] label code that will need to change for editable metadata --- historyKeeper.js | 16 ++++++++-------- rpc.js | 2 +- storage/file.js | 6 +++++- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/historyKeeper.js b/historyKeeper.js index 03c767506..169b14ffb 100644 --- a/historyKeeper.js +++ b/historyKeeper.js @@ -61,13 +61,13 @@ module.exports.create = function (cfg) { const cpIndex = []; let messageBuf = []; let validateKey; - let metadata; + let metadata; // FIXME METADATA let i = 0; store.readMessagesBin(channelName, 0, (msgObj, rmcb) => { let msg; i++; if (!validateKey && msgObj.buff.indexOf('validateKey') > -1) { - metadata = msg = tryParse(msgObj.buff.toString('utf8')); + metadata = msg = tryParse(msgObj.buff.toString('utf8')); // FIXME METADATA if (typeof msg === "undefined") { return rmcb(); } if (msg.validateKey) { validateKey = historyKeeperKeys[channelName] = msg; @@ -105,7 +105,7 @@ module.exports.create = function (cfg) { cpIndex: sliceCpIndex(cpIndex, i), offsetByHash: offsetByHash, size: size, - metadata: metadata, + metadata: metadata, // FIXME METADATA line: i }); }); @@ -438,16 +438,16 @@ module.exports.create = function (cfg) { so, let's just fall through... */ if (err) { return w(); } - if (!index || !index.metadata) { return void w(); } + if (!index || !index.metadata) { return void w(); } // FIXME METADATA // Store the metadata if we don't have it in memory if (!historyKeeperKeys[channelName]) { - historyKeeperKeys[channelName] = index.metadata; + historyKeeperKeys[channelName] = index.metadata; // FIXME METADATA } // And then check if the channel is expired. If it is, send the error and abort if (checkExpired(ctx, channelName)) { return void waitFor.abort(); } // Send the metadata to the user if (!lastKnownHash && index.cpIndex.length > 1) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); // FIXME METADATA return; } w(); @@ -494,8 +494,8 @@ module.exports.create = function (cfg) { key.expire = expire; } historyKeeperKeys[channelName] = key; - storeMessage(ctx, chan, JSON.stringify(key), false, undefined); - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(key)]); + storeMessage(ctx, chan, JSON.stringify(key), false, undefined); // FIXME METADATA + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(key)]); // FIXME METADATA } // End of history message: diff --git a/rpc.js b/rpc.js index a69033cc1..9fb431708 100644 --- a/rpc.js +++ b/rpc.js @@ -1722,7 +1722,7 @@ RPC.create = function ( respond(e, [null, size, null]); }); case 'GET_METADATA': - return void getMetadata(Env, msg[1], function (e, data) { + return void getMetadata(Env, msg[1], function (e, data) { // FIXME METADATA WARN(e, msg[1]); respond(e, [null, data, null]); }); diff --git a/storage/file.js b/storage/file.js index 99272fc6f..196a83b32 100644 --- a/storage/file.js +++ b/storage/file.js @@ -60,6 +60,7 @@ var getMetadataAtPath = function (Env, path, cb) { stream.on('error', function (e) { complete(e); }); }; +// FIXME METADATA var getChannelMetadata = function (Env, channelId, cb) { var path = mkPath(Env, channelId); getMetadataAtPath(Env, path, cb); @@ -255,6 +256,9 @@ var listChannels = function (root, handler, cb) { var wait = w(); dirList.forEach(function (dir) { sema.take(function (give) { + // TODO modify the asynchronous bits here to keep less in memory at any given time + // list a directory -> process its contents with semaphores until less than N jobs are running + // then list the next directory... var nestedDirPath = Path.join(root, dir); Fs.readdir(nestedDirPath, w(give(function (err, list) { if (err) { return void handler(err); } // Is this correct? @@ -600,7 +604,7 @@ module.exports.create = function ( if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } channelBytes(env, channelName, cb); }, - getChannelMetadata: function (channelName, cb) { + getChannelMetadata: function (channelName, cb) { // FIXME METADATA if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } getChannelMetadata(env, channelName, cb); }, From 3513e437b6871b4f1ddd77c70d2690b4dac2af52 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 25 Jun 2019 15:38:56 +0200 Subject: [PATCH 002/258] update atime when accessing files --- storage/file.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/file.js b/storage/file.js index 196a83b32..0eb71f284 100644 --- a/storage/file.js +++ b/storage/file.js @@ -470,9 +470,8 @@ const messageBin = (env, chanName, msgBin, cb) => { chan.writeStream.write(msgBin, function () { /*::if (!chan) { throw new Error("Flow unreachable"); }*/ chan.onError.splice(chan.onError.indexOf(complete), 1); + chan.atime = +new Date(); if (!cb) { return; } - //chan.messages.push(msg); - chan.atime = +new Date(); // FIXME seems like odd behaviour that not passing a callback would result in not updating atime... complete(); }); }); From fc9aaf0ecb1e753a76b7bdd2072311057c27ad0e Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 25 Jun 2019 15:40:03 +0200 Subject: [PATCH 003/258] remove unnecessary argument from channelExists also add some notes --- storage/file.js | 50 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/storage/file.js b/storage/file.js index 0eb71f284..6b1285867 100644 --- a/storage/file.js +++ b/storage/file.js @@ -27,6 +27,29 @@ var mkArchivePath = function (env, channelId) { return Path.join(env.archiveRoot, 'datastore', channelId.slice(0, 2), channelId) + '.ndjson'; }; +var mkMetadataPath = function (env, channelId) { + return Path.join(env.root, channelId.slice(0, 2), channelId) + '.metadata.ndjson'; +}; + +var mkArchiveMetadataPath = function (env, channelId) { + return Path.join(env.archiveRoot, 'datastore', channelId.slice(0, 2), channelId) + '.metadata.ndjson'; +}; + +// pass in the path so we can reuse the same function for archived files +var channelExists = function (filepath, cb) { + Fs.stat(filepath, function (err, stat) { + if (err) { + if (err.code === 'ENOENT') { + // no, the file doesn't exist + return void cb(void 0, false); + } + return void cb(err); + } + if (!stat.isFile()) { return void cb("E_NOT_FILE"); } + return void cb(void 0, true); + }); +}; + var getMetadataAtPath = function (Env, path, cb) { var remainder = ''; var stream = Fs.createReadStream(path, { encoding: 'utf8' }); @@ -80,6 +103,7 @@ var closeChannel = function (env, channelName, cb) { var clearChannel = function (env, channelId, cb) { var path = mkPath(env, channelId); + // FIXME METADATA getMetadataAtPath(env, path, function (e, metadata) { if (e) { return cb(new Error(e)); } if (!metadata) { @@ -209,31 +233,21 @@ var checkPath = function (path, callback) { }); }; +// FIXME METADATA +// remove associated metadata as well var removeChannel = function (env, channelName, cb) { var filename = mkPath(env, channelName); Fs.unlink(filename, cb); }; -// pass in the path so we can reuse the same function for archived files -var channelExists = function (filepath, channelName, cb) { - Fs.stat(filepath, function (err, stat) { - if (err) { - if (err.code === 'ENOENT') { - // no, the file doesn't exist - return void cb(void 0, false); - } - return void cb(err); - } - if (!stat.isFile()) { return void cb("E_NOT_FILE"); } - return void cb(void 0, true); - }); -}; - +// FIXME +// remove associated metadata as well var removeArchivedChannel = function (env, channelName, cb) { var filename = mkArchivePath(env, channelName); Fs.unlink(filename, cb); }; +// TODO confirm that we're ignoring metadata files var listChannels = function (root, handler, cb) { // do twenty things at a time var sema = Semaphore.create(20); @@ -364,6 +378,8 @@ var channelBytes = function (env, chanName, cb) { }); }; +// FIXME METADATA +// implement metadata bytes as well? /*:: export type ChainPadServer_ChannelInternal_t = { atime: number, @@ -618,13 +634,13 @@ module.exports.create = function ( if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } // construct the path var filepath = mkPath(env, channelName); - channelExists(filepath, channelName, cb); + channelExists(filepath, cb); }, isChannelArchived: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } // construct the path var filepath = mkArchivePath(env, channelName); - channelExists(filepath, channelName, cb); + channelExists(filepath, cb); }, listArchivedChannels: function (handler, cb) { listChannels(Path.join(env.archiveRoot, 'datastore'), handler, cb); From 0668abf68c0c3c548ed8a5e68d5846f661a8bf4f Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 25 Jun 2019 15:42:10 +0200 Subject: [PATCH 004/258] correct the regular expression for iterating over channels --- storage/file.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/file.js b/storage/file.js index 6b1285867..f7484a340 100644 --- a/storage/file.js +++ b/storage/file.js @@ -279,7 +279,7 @@ var listChannels = function (root, handler, cb) { list.forEach(function (item) { // ignore things that don't match the naming pattern - if (/^\./.test(item) || !/[0-9a-fA-F]{32,}\.ndjson$/.test(item)) { return; } + if (/^\./.test(item) || !/[0-9a-fA-F]{32}\.ndjson$/.test(item)) { return; } var filepath = Path.join(nestedDirPath, item); var channel = filepath.replace(/\.ndjson$/, '').replace(/.*\//, ''); if ([32, 34].indexOf(channel.length) === -1) { return; } From a16d7171ad2cdcc3427c3ac173836fd6dd9c4bda Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 25 Jun 2019 16:31:21 +0200 Subject: [PATCH 005/258] add some more notes --- historyKeeper.js | 9 +++++++++ storage/file.js | 3 +-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/historyKeeper.js b/historyKeeper.js index 169b14ffb..01786c6e4 100644 --- a/historyKeeper.js +++ b/historyKeeper.js @@ -581,6 +581,15 @@ module.exports.create = function (cfg) { if (msg[3] === 'CLEAR_OWNED_CHANNEL') { onChannelCleared(ctx, msg[4]); } + + // FIXME METADATA + /* + if (msg[3] === 'SET_METADATA') { // or whatever we call the RPC???? + + } + + */ + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify([parsed[0]].concat(output))]); }); } catch (e) { diff --git a/storage/file.js b/storage/file.js index f7484a340..285f64dd0 100644 --- a/storage/file.js +++ b/storage/file.js @@ -197,7 +197,6 @@ const mkOffsetCounter = () => { const readMessagesBin = (env, id, start, msgHandler, cb) => { const stream = Fs.createReadStream(mkPath(env, id), { start: start }); - // TODO get the channel and add the atime let keepReading = true; Pull( ToPull.read(stream), @@ -213,7 +212,6 @@ const readMessagesBin = (env, id, start, msgHandler, cb) => { }; var checkPath = function (path, callback) { - // TODO check if we actually need to use stat at all Fs.stat(path, function (err) { if (!err) { callback(undefined, true); @@ -314,6 +312,7 @@ var listChannels = function (root, handler, cb) { // move a channel's log file from its current location // to an equivalent location in the cold storage directory var archiveChannel = function (env, channelName, cb) { + // TODO close channels before archiving them? if (!env.retainData) { return void cb("ARCHIVES_DISABLED"); } From fd43da2dbf786a1f4496ba155c7f3f8f3fa95e94 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 25 Jun 2019 16:38:29 +0200 Subject: [PATCH 006/258] archive and restore metadata logs and channel logs together --- storage/file.js | 89 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 4 deletions(-) diff --git a/storage/file.js b/storage/file.js index 285f64dd0..6a7e0a17f 100644 --- a/storage/file.js +++ b/storage/file.js @@ -6,6 +6,7 @@ var Fse = require("fs-extra"); var Path = require("path"); var nThen = require("nthen"); var Semaphore = require("saferphore"); +var Once = require("../lib/once"); const ToPull = require('stream-to-pull-stream'); const Pull = require('pull-stream'); @@ -331,20 +332,100 @@ var archiveChannel = function (env, channelName, cb) { // use Fse.move to move it, Fse makes paths to the directory when you use it. // https://github.com/jprichardson/node-fs-extra/blob/HEAD/docs/move.md - Fse.move(currentPath, archivePath, { overwrite: true }, cb); + nThen(function (w) { + // move the channel log and abort if anything goes wrong + Fse.move(currentPath, archivePath, { overwrite: true }, w(function (err) { + if (!err) { return; } + w.abort(); + cb(err); + })); + }).nThen(function (w) { + // archive the dedicated metadata channel + var metadataPath = mkMetadataPath(env, channelName); + var archiveMetadataPath = mkArchiveMetadataPath(env, channelName); + + Fse.move(metadataPath, archiveMetadataPath, { overwrite: true, }, w(function (err) { + // there's no metadata to archive, so you're done! + if (err && err.code === "ENOENT") { + return void cb(); + } + + // there was an error archiving the metadata + if (err) { + return void cb("E_METADATA_ARCHIVAL" + (err.code ? "_" + err.code: '')); + } + + // it was archived successfully + cb(); + })); + }); }; var unarchiveChannel = function (env, channelName, cb) { // very much like 'archiveChannel' but in the opposite direction // the file is currently archived - var currentPath = mkArchivePath(env, channelName); - var unarchivedPath = mkPath(env, channelName); + var channelPath = mkArchivePath(env, channelName); + var metadataPath = mkMetadataPath(env, channelName); + + // don't call the callback multiple times + var CB = Once(cb); // if a file exists in the unarchived path, you probably don't want to clobber its data // so unlike 'archiveChannel' we won't overwrite. // Fse.move will call back with EEXIST in such a situation - Fse.move(currentPath, unarchivedPath, cb); + + nThen(function (w) { + // if either metadata or a file exist in prod, abort + channelExists(channelPath, w(function (err, exists) { + if (err) { + w.abort(); + return void CB(err); + } + if (exists) { + w.abort(); + return CB('UNARCHIVE_CHANNEL_CONFLICT'); + } + })); + channelExists(metadataPath, w(function (err, exists) { + if (err) { + w.abort(); + return void CB(err); + } + if (exists) { + w.abort(); + return CB("UNARCHIVE_METADATA_CONFLICT"); + } + })); + }).nThen(function (w) { + // construct archive paths + var archiveChannelPath = mkArchivePath(env, channelName); + // restore the archived channel + Fse.move(archiveChannelPath, channelPath, w(function (err) { + if (err) { + w.abort(); + return void CB(err); + } + })); + }).nThen(function (w) { + var archiveMetadataPath = mkArchiveMetadataPath(env, channelName); + // TODO validate that it's ok to move metadata non-atomically + + // restore the metadata log + Fse.move(archiveMetadataPath, metadataPath, w(function (err) { + // if there's nothing to move, you're done. + if (err && err.code === 'ENOENT') { + return CB(); + } + // call back with an error if something goes wrong + if (err) { + w.abort(); + return void CB("E_METADATA_RESTORATION" + (err.code ? "_" + err.code: "")); + } + // otherwise it was moved successfully + CB(); + })); + }); }; var flushUnusedChannels = function (env, cb, frame) { From b128b7f02c9381d1d6c375ed53c0b357b148a23d Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 26 Jun 2019 17:55:36 +0200 Subject: [PATCH 007/258] updated methods for working with editable metadata --- storage/file.js | 233 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 188 insertions(+), 45 deletions(-) diff --git a/storage/file.js b/storage/file.js index 6a7e0a17f..c54db236d 100644 --- a/storage/file.js +++ b/storage/file.js @@ -84,12 +84,6 @@ var getMetadataAtPath = function (Env, path, cb) { stream.on('error', function (e) { complete(e); }); }; -// FIXME METADATA -var getChannelMetadata = function (Env, channelId, cb) { - var path = mkPath(Env, channelId); - getMetadataAtPath(Env, path, cb); -}; - var closeChannel = function (env, channelName, cb) { if (!env.channels[channelName]) { return void cb(); } try { @@ -104,7 +98,6 @@ var closeChannel = function (env, channelName, cb) { var clearChannel = function (env, channelId, cb) { var path = mkPath(env, channelId); - // FIXME METADATA getMetadataAtPath(env, path, function (e, metadata) { if (e) { return cb(new Error(e)); } if (!metadata) { @@ -153,6 +146,77 @@ var readMessages = function (path, msgHandler, cb) { stream.on('error', function (e) { complete(e); }); }; +// FIXME METADATA +// XXX deprecate this everywhere in favour of the new method +var getChannelMetadata = function (Env, channelId, cb) { + var path = mkPath(Env, channelId); + + // gets metadata embedded in a file + getMetadataAtPath(Env, path, cb); +}; + +var readMetadata = function (env, channelId, handler, cb) { +/* + +Possibilities + + 1. there is no metadata because it's an old channel + 2. there is metadata in the first line of the channel, but nowhere else + 3. there is metadata in the first line of the channel as well as in a dedicated log + 4. there is no metadata in the first line of the channel. Everything is in the dedicated log + +How to proceed + + 1. load the first line of the channel and treat it as a metadata message if applicable + 2. load the dedicated log and treat it as an update + +*/ + + var CB = Once(cb); + + var index = 0; + nThen(function (w) { + // returns the first line of a channel, parsed... + getChannelMetadata(env, channelId, w(function (err, data) { + if (err) { + // 'INVALID_METADATA' if it can't parse + // stream errors if anything goes wrong at a lower level + // ENOENT (no channel here) + return void handler(err, undefined, index++); + } + // disregard anything that isn't a map + if (!data || typeof(data) !== 'object' || Array.isArray(data)) { return; } + + // otherwise it's good. + handler(null, data, index++); + })); + }).nThen(function (w) { + var metadataPath = mkMetadataPath(env, channelId); + readMessages(metadataPath, function (line) { + if (!line) { return; } + try { + var parsed = JSON.parse(line); + handler(null, parsed, index++); + } catch (e) { + handler(e, line, index++); + } + }, w(function (err) { + if (err) { + // ENOENT => there is no metadata log + if (err.code === 'ENOENT') { return void CB(); } + // otherwise stream errors? + CB(err); + } + CB(); + })); + }); +}; + +var writeMetadata = function (env, channelId, data, cb) { + cb = cb; + // XXX +}; + const NEWLINE_CHR = ('\n').charCodeAt(0); const mkBufferSplit = () => { let remainder = null; @@ -232,20 +296,62 @@ var checkPath = function (path, callback) { }); }; -// FIXME METADATA -// remove associated metadata as well +var labelError = function (label, err) { + return label + (err.code ? "_" + err.code: ''); +}; + var removeChannel = function (env, channelName, cb) { - var filename = mkPath(env, channelName); - Fs.unlink(filename, cb); + var channelPath = mkPath(env, channelName); + var metadataPath = mkMetadataPath(env, channelName); + + var CB = Once(cb); + + nThen(function (w) { + Fs.unlink(channelPath, w(function (err) { + if (err) { + w.abort(); + CB(labelError("E_CHANNEL_REMOVAL", err)); + } + })); + Fs.unlink(metadataPath, w(function (err) { + if (err) { + if (err.code === 'ENOENT') { return; } // proceed if there's no metadata to delete + w.abort(); + CB(labelError("E_METADATA_REMOVAL", err)); + } + })); + }).nThen(function () { + CB(); + }); }; -// FIXME -// remove associated metadata as well var removeArchivedChannel = function (env, channelName, cb) { - var filename = mkArchivePath(env, channelName); - Fs.unlink(filename, cb); + var channelPath = mkArchivePath(env, channelName); + var metadataPath = mkArchiveMetadataPath(env, channelName); + + var CB = Once(cb); + + nThen(function (w) { + Fs.unlink(channelPath, w(function (err) { + if (err) { + w.abort(); + CB(labelError("E_ARCHIVED_CHANNEL_REMOVAL", err)); + } + })); + Fs.unlink(metadataPath, w(function (err) { + if (err) { + if (err.code === "ENOENT") { return; } + w.abort(); + CB(labelError("E_ARCHIVED_METADATA_REMOVAL", err)); + } + })); + }).nThen(function () { + CB(); + }); }; +// TODO implement a method of removing metadata that doesn't have a corresponding channel + // TODO confirm that we're ignoring metadata files var listChannels = function (root, handler, cb) { // do twenty things at a time @@ -352,7 +458,7 @@ var archiveChannel = function (env, channelName, cb) { // there was an error archiving the metadata if (err) { - return void cb("E_METADATA_ARCHIVAL" + (err.code ? "_" + err.code: '')); + return void cb(labelError("E_METADATA_ARCHIVAL", err)); } // it was archived successfully @@ -420,7 +526,7 @@ var unarchiveChannel = function (env, channelName, cb) { // call back with an error if something goes wrong if (err) { w.abort(); - return void CB("E_METADATA_RESTORATION" + (err.code ? "_" + err.code: "")); + return void CB(labelError("E_METADATA_RESTORATION", err)); } // otherwise it was moved successfully CB(); @@ -458,7 +564,6 @@ var channelBytes = function (env, chanName, cb) { }); }; -// FIXME METADATA // implement metadata bytes as well? /*:: export type ChainPadServer_ChannelInternal_t = { @@ -662,80 +767,118 @@ module.exports.create = function ( })); }).nThen(function () { cb({ - readMessagesBin: (channelName, start, asyncMsgHandler, cb) => { - if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - readMessagesBin(env, channelName, start, asyncMsgHandler, cb); - }, + // OLDER METHODS + // write a new message to a log message: function (channelName, content, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } message(env, channelName, content, cb); }, + // iterate over all the messages in a log + getMessages: function (channelName, msgHandler, cb) { + if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } + getMessages(env, channelName, msgHandler, cb); + }, + + // NEWER IMPLEMENTATIONS OF THE SAME THING + // write a new message to a log messageBin: (channelName, content, cb) => { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } messageBin(env, channelName, content, cb); }, - getMessages: function (channelName, msgHandler, cb) { + // iterate over the messages in a log + readMessagesBin: (channelName, start, asyncMsgHandler, cb) => { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - getMessages(env, channelName, msgHandler, cb); + readMessagesBin(env, channelName, start, asyncMsgHandler, cb); }, + + // METHODS for deleting data + // remove a channel and its associated metadata log if present removeChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } removeChannel(env, channelName, function (err) { cb(err); }); }, + // remove a channel and its associated metadata log from the archive directory removeArchivedChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } removeArchivedChannel(env, channelName, cb); }, - closeChannel: function (channelName, cb) { - if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - closeChannel(env, channelName, cb); - }, - flushUnusedChannels: function (cb) { - flushUnusedChannels(env, cb); - }, - getChannelSize: function (channelName, cb) { - if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - channelBytes(env, channelName, cb); - }, - getChannelMetadata: function (channelName, cb) { // FIXME METADATA - if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } - getChannelMetadata(env, channelName, cb); - }, + // clear all data for a channel but preserve its metadata clearChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } clearChannel(env, channelName, cb); }, - listChannels: function (handler, cb) { - listChannels(env.root, handler, cb); - }, + + // check if a channel exists in the database isChannelAvailable: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } // construct the path var filepath = mkPath(env, channelName); channelExists(filepath, cb); }, + // check if a channel exists in the archive isChannelArchived: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } // construct the path var filepath = mkArchivePath(env, channelName); channelExists(filepath, cb); }, - listArchivedChannels: function (handler, cb) { - listChannels(Path.join(env.archiveRoot, 'datastore'), handler, cb); - }, + // move a channel from the database to the archive, along with its metadata archiveChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } archiveChannel(env, channelName, cb); }, + // restore a channel from the archive to the database, along with its metadata restoreArchivedChannel: function (channelName, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } unarchiveChannel(env, channelName, cb); }, + + // METADATA METHODS + // fetch the metadata for a channel + getChannelMetadata: function (channelName, cb) { + if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } + getChannelMetadata(env, channelName, cb); + }, + // iterate over multiple lines of metadata changes + readChannelMetadata: function (channelName, handler, cb) { + if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } + readMetadata(env, channelName, handler, cb); + }, + // write a new line to a metadata log + writeMetadata: function (channelName, data, cb) { + if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } + writeMetadata(env, channelName, data, cb); + }, + + // CHANNEL ITERATION + listChannels: function (handler, cb) { + listChannels(env.root, handler, cb); + }, + listArchivedChannels: function (handler, cb) { + listChannels(Path.join(env.archiveRoot, 'datastore'), handler, cb); + }, + + getChannelSize: function (channelName, cb) { + if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } + channelBytes(env, channelName, cb); + }, + // OTHER DATABASE FUNCTIONALITY + // remove a particular channel from the cache + closeChannel: function (channelName, cb) { + if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } + closeChannel(env, channelName, cb); + }, + // iterate over open channels and close any that are not active + flushUnusedChannels: function (cb) { + flushUnusedChannels(env, cb); + }, + // write to a log file log: function (channelName, content, cb) { message(env, channelName, content, cb); }, + // shut down the database shutdown: function () { clearInterval(it); } From 7b18bf91ce3ca263aa1eba3ddad5b337d666a7f2 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 27 Jun 2019 11:11:22 +0200 Subject: [PATCH 008/258] implement functions for iterating over metadata logs --- lib/metadata.js | 99 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 lib/metadata.js diff --git a/lib/metadata.js b/lib/metadata.js new file mode 100644 index 000000000..b4e9f918c --- /dev/null +++ b/lib/metadata.js @@ -0,0 +1,99 @@ +var Meta = module.exports; + +/* Metadata fields: + + * channel + * validateKey + * owners + * ADD_OWNERS + * RM_OWNERS + * expire + +*/ + +var commands = {}; + +// ["ADD_OWNERS", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I="], 1561623438989] +commands.ADD_OWNERS = function (meta, args) { + if (!Array.isArray(args)) { + throw new Error('METADATA_INVALID_OWNERS'); + } + if (!Array.isArray(meta.owners)) { + throw new Error("METADATA_NONSENSE_OWNERS"); + } + + args.forEach(function (owner) { + if (meta.owners.indexOf(owner) >= 0) { return; } + meta.owners.push(owner); + }); +}; + +// ["RM_OWNERS", ["CrufexqXcY-z+eKJlEbNELVy5Sb7E-EAAEFI8GnEtZ0="], 1561623439989] +commands.RM_OWNERS = function (meta, args) { + if (!Array.isArray(args)) { + throw new Error('METADATA_INVALID_OWNERS'); + } + if (!Array.isArray(meta.owners)) { + throw new Error("METADATA_NONSENSE_OWNERS"); + } + + args.forEach(function (owner) { + var index = meta.owners.indexOf(owner); + meta.owners.splice(index, 1); + }); +}; + +commands.UPDATE_EXPIRATION = function () { + +}; + +var handleCommand = function (meta, line) { + var command = line[0]; + var args = line[1]; + //var time = line[2]; + + if (typeof(commands[command]) !== 'function') { + throw new Error("METADATA_UNSUPPORTED_COMMAND"); + } + + commands[command](meta, args); +}; + +Meta.createLineHandler = function (ref, errorHandler) { + ref.meta = {}; + + errorHandler = errorHandler; + + return function (err, line, index) { + if (err) { + return void errorHandler('METADATA_HANDLER_LINE_ERR', { + error: err, + index: index, + line: JSON.stringify(line), + }); + } + + if (Array.isArray(line)) { + try { + handleCommand(ref.meta, line); + } catch (err) { + errorHandler("METADATA_COMMAND_ERR", { + error: err.stack, + line: line, + }); + } + return; + } + + if (index === 0 && typeof(line) === 'object') { + // special case! + ref.meta = line; + return; + } + + errorHandler("METADATA_HANDLER_WEIRDLINE", { + line: line, + index: index + }); + }; +}; From efd0efede42ed095f696493ecc54156a8f754156 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 27 Jun 2019 12:13:29 +0200 Subject: [PATCH 009/258] implement RESET_OWNERS command for editable metadata --- lib/metadata.js | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/metadata.js b/lib/metadata.js index b4e9f918c..9ef939032 100644 --- a/lib/metadata.js +++ b/lib/metadata.js @@ -15,9 +15,13 @@ var commands = {}; // ["ADD_OWNERS", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I="], 1561623438989] commands.ADD_OWNERS = function (meta, args) { + // bail out if args isn't an array if (!Array.isArray(args)) { throw new Error('METADATA_INVALID_OWNERS'); } + + // you shouldn't be able to get here if there are no owners + // because only an owner should be able to change the owners if (!Array.isArray(meta.owners)) { throw new Error("METADATA_NONSENSE_OWNERS"); } @@ -30,19 +34,39 @@ commands.ADD_OWNERS = function (meta, args) { // ["RM_OWNERS", ["CrufexqXcY-z+eKJlEbNELVy5Sb7E-EAAEFI8GnEtZ0="], 1561623439989] commands.RM_OWNERS = function (meta, args) { + // what are you doing if you don't have owners to remove? if (!Array.isArray(args)) { throw new Error('METADATA_INVALID_OWNERS'); } + // if there aren't any owners to start, this is also pointless if (!Array.isArray(meta.owners)) { throw new Error("METADATA_NONSENSE_OWNERS"); } + // remove owners one by one + // we assume there are no duplicates args.forEach(function (owner) { var index = meta.owners.indexOf(owner); + if (index < 0) { return; } meta.owners.splice(index, 1); }); }; +// ["RESET_OWNERS", ["7eEqelGso3EBr5jHlei6av4r9w2B9XZiGGwA1EgZ-5I="], 1561623439989] +commands.RESET_OWNERS = function (meta, args) { + // expect a new array, even if it's empty + if (!Array.isArray(args)) { + throw new Error('METADATA_INVALID_OWNERS'); + } + // assume there are owners to start + if (!Array.isArray(meta.owners)) { + throw new Error("METADATA_NONSENSE_OWNERS"); + } + + // overwrite the existing owners with the new one + meta.owners = args; +}; + commands.UPDATE_EXPIRATION = function () { }; From b36f50f2c975c0c384a5dd315ba6c63bce89c25e Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 27 Jun 2019 18:21:12 +0200 Subject: [PATCH 010/258] update metadata queries to include edits --- lib/metadata.js | 13 ++++++----- rpc.js | 38 ++++++++++++++++++++++---------- storage/file.js | 58 ++++++++++++++++++++++++++++++------------------- 3 files changed, 70 insertions(+), 39 deletions(-) diff --git a/lib/metadata.js b/lib/metadata.js index 9ef939032..187da029e 100644 --- a/lib/metadata.js +++ b/lib/metadata.js @@ -86,9 +86,8 @@ var handleCommand = function (meta, line) { Meta.createLineHandler = function (ref, errorHandler) { ref.meta = {}; - errorHandler = errorHandler; - - return function (err, line, index) { + var index = 0; + return function (err, line) { if (err) { return void errorHandler('METADATA_HANDLER_LINE_ERR', { error: err, @@ -98,11 +97,12 @@ Meta.createLineHandler = function (ref, errorHandler) { } if (Array.isArray(line)) { + index++; try { handleCommand(ref.meta, line); - } catch (err) { + } catch (err2) { errorHandler("METADATA_COMMAND_ERR", { - error: err.stack, + error: err2.stack, line: line, }); } @@ -110,6 +110,7 @@ Meta.createLineHandler = function (ref, errorHandler) { } if (index === 0 && typeof(line) === 'object') { + index++; // special case! ref.meta = line; return; @@ -117,7 +118,7 @@ Meta.createLineHandler = function (ref, errorHandler) { errorHandler("METADATA_HANDLER_WEIRDLINE", { line: line, - index: index + index: index++, }); }; }; diff --git a/rpc.js b/rpc.js index 9fb431708..07d4ae313 100644 --- a/rpc.js +++ b/rpc.js @@ -313,22 +313,33 @@ var getFileSize = function (Env, channel, cb) { }); }; +var Meta = require("./lib/metadata"); + var getMetadata = function (Env, channel, cb) { if (!isValidId(channel)) { return void cb('INVALID_CHAN'); } - if (channel.length === 32) { - if (typeof(Env.msgStore.getChannelMetadata) !== 'function') { - return cb('GET_CHANNEL_METADATA_UNSUPPORTED'); + if (channel.length !== 32) { return cb("INVALID_CHAN"); } + + var ref = {}; + var lineHandler = Meta.createLineHandler(ref, Log.error); + + return void Env.msgStore.readChannelMetadata(channel, lineHandler, function (err) { + if (err) { + // stream errors? + return void cb(err); } + cb(void 0, ref.meta); + }); - return void Env.msgStore.getChannelMetadata(channel, function (e, data) { - if (e) { - if (e.code === 'INVALID_METADATA') { return void cb(void 0, {}); } - return void cb(e.code); - } - cb(void 0, data); - }); - } +/* + // FIXME METADATA + return void Env.msgStore.getChannelMetadata(channel, function (e, data) { + if (e) { + if (e.code === 'INVALID_METADATA') { return void cb(void 0, {}); } + return void cb(e.code); + } + cb(void 0, data); + });*/ }; var getMultipleFileSize = function (Env, channels, cb) { @@ -802,10 +813,12 @@ var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { return cb('INVALID_ARGUMENTS'); } + // FIXME METADATA if (!(Env.msgStore && Env.msgStore.getChannelMetadata)) { return cb('E_NOT_IMPLEMENTED'); } + // FIXME METADATA Env.msgStore.getChannelMetadata(channelId, function (e, metadata) { if (e) { return cb(e); } if (!(metadata && Array.isArray(metadata.owners))) { return void cb('E_NO_OWNERS'); } @@ -822,6 +835,7 @@ var clearOwnedChannel = function (Env, channelId, unsafeKey, cb) { }; var removeOwnedBlob = function (Env, blobId, unsafeKey, cb) { + // FIXME METADATA var safeKey = escapeKeyCharacters(unsafeKey); var safeKeyPrefix = safeKey.slice(0,3); var blobPrefix = blobId.slice(0,2); @@ -891,10 +905,12 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { return void removeOwnedBlob(Env, channelId, unsafeKey, cb); } + // FIXME METADATA if (!(Env.msgStore && Env.msgStore.removeChannel && Env.msgStore.getChannelMetadata)) { return cb("E_NOT_IMPLEMENTED"); } + // FIXME METADATA Env.msgStore.getChannelMetadata(channelId, function (e, metadata) { if (e) { return cb(e); } if (!(metadata && Array.isArray(metadata.owners))) { return void cb('E_NO_OWNERS'); } diff --git a/storage/file.js b/storage/file.js index c54db236d..bf056bc3c 100644 --- a/storage/file.js +++ b/storage/file.js @@ -155,6 +155,28 @@ var getChannelMetadata = function (Env, channelId, cb) { getMetadataAtPath(Env, path, cb); }; +// low level method for getting just the dedicated metadata channel +var getDedicatedMetadata = function (env, channelId, handler, cb) { + var metadataPath = mkMetadataPath(env, channelId); + readMessages(metadataPath, function (line) { + if (!line) { return; } + try { + var parsed = JSON.parse(line); + handler(null, parsed); + } catch (e) { + handler(e, line); + } + }, function (err) { + if (err) { + // ENOENT => there is no metadata log + if (err.code === 'ENOENT') { return void cb(); } + // otherwise stream errors? + cb(err); + } + cb(); + }); +}; + var readMetadata = function (env, channelId, handler, cb) { /* @@ -172,9 +194,6 @@ How to proceed */ - var CB = Once(cb); - - var index = 0; nThen(function (w) { // returns the first line of a channel, parsed... getChannelMetadata(env, channelId, w(function (err, data) { @@ -182,33 +201,22 @@ How to proceed // 'INVALID_METADATA' if it can't parse // stream errors if anything goes wrong at a lower level // ENOENT (no channel here) - return void handler(err, undefined, index++); + return void handler(err); } // disregard anything that isn't a map if (!data || typeof(data) !== 'object' || Array.isArray(data)) { return; } // otherwise it's good. - handler(null, data, index++); + handler(null, data); })); - }).nThen(function (w) { - var metadataPath = mkMetadataPath(env, channelId); - readMessages(metadataPath, function (line) { - if (!line) { return; } - try { - var parsed = JSON.parse(line); - handler(null, parsed, index++); - } catch (e) { - handler(e, line, index++); - } - }, w(function (err) { + }).nThen(function () { + getDedicatedMetadata(env, channelId, handler, function (err) { if (err) { - // ENOENT => there is no metadata log - if (err.code === 'ENOENT') { return void CB(); } - // otherwise stream errors? - CB(err); + // stream errors? + return void cb(err); } - CB(); - })); + cb(); + }); }); }; @@ -841,6 +849,12 @@ module.exports.create = function ( if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } getChannelMetadata(env, channelName, cb); }, + // iterate over lines of metadata changes from a dedicated log + readDedicatedMetadata: function (channelName, handler, cb) { + if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } + getDedicatedMetadata(env, channelName, handler, cb); + }, + // iterate over multiple lines of metadata changes readChannelMetadata: function (channelName, handler, cb) { if (!isValidChannelId(channelName)) { return void cb(new Error('EINVAL')); } From 6c907dbc3218da38f112db5be76770ccfc735fb0 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 27 Jun 2019 18:22:06 +0200 Subject: [PATCH 011/258] add more specific comments --- historyKeeper.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/historyKeeper.js b/historyKeeper.js index 01786c6e4..dac280ffb 100644 --- a/historyKeeper.js +++ b/historyKeeper.js @@ -400,7 +400,7 @@ module.exports.create = function (cfg) { var lastKnownHash = parsed[3]; var owners; var expire; - if (parsed[2] && typeof parsed[2] === "object") { + if (parsed[2] && typeof parsed[2] === "object") { // FIXME METADATA RECEIVE validateKey = parsed[2].validateKey; lastKnownHash = parsed[2].lastKnownHash; owners = parsed[2].owners; @@ -438,16 +438,16 @@ module.exports.create = function (cfg) { so, let's just fall through... */ if (err) { return w(); } - if (!index || !index.metadata) { return void w(); } // FIXME METADATA + if (!index || !index.metadata) { return void w(); } // FIXME METADATA READ // Store the metadata if we don't have it in memory if (!historyKeeperKeys[channelName]) { - historyKeeperKeys[channelName] = index.metadata; // FIXME METADATA + historyKeeperKeys[channelName] = index.metadata; // FIXME METADATA STORE } // And then check if the channel is expired. If it is, send the error and abort if (checkExpired(ctx, channelName)) { return void waitFor.abort(); } // Send the metadata to the user if (!lastKnownHash && index.cpIndex.length > 1) { - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); // FIXME METADATA + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(index.metadata)], w); // FIXME METADATA SEND return; } w(); @@ -480,6 +480,9 @@ module.exports.create = function (cfg) { // If this is a new channel, we need to store the metadata as // the first message in the file + // FIXME METADATA COMMENT + // in fact it should be written to a new metadata log + // XXX read this kind of metadata before writing it const chan = ctx.channels[channelName]; if (msgCount === 0 && !historyKeeperKeys[channelName] && chan && chan.indexOf(user) > -1) { var key = {}; @@ -494,8 +497,8 @@ module.exports.create = function (cfg) { key.expire = expire; } historyKeeperKeys[channelName] = key; - storeMessage(ctx, chan, JSON.stringify(key), false, undefined); // FIXME METADATA - sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(key)]); // FIXME METADATA + storeMessage(ctx, chan, JSON.stringify(key), false, undefined); // FIXME METADATA WRITE + sendMsg(ctx, user, [0, HISTORY_KEEPER_ID, 'MSG', user.id, JSON.stringify(key)]); // FIXME METADATA SEND } // End of history message: @@ -582,7 +585,7 @@ module.exports.create = function (cfg) { onChannelCleared(ctx, msg[4]); } - // FIXME METADATA + // FIXME METADATA CHANGE /* if (msg[3] === 'SET_METADATA') { // or whatever we call the RPC???? From 92dda92a0035ed0795c56b66f952217b69550c95 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 4 Jul 2019 11:04:28 +0200 Subject: [PATCH 012/258] very WIP update to serve accumulated metadata updates --- historyKeeper.js | 101 ++++++++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 36 deletions(-) diff --git a/historyKeeper.js b/historyKeeper.js index dac280ffb..37d647360 100644 --- a/historyKeeper.js +++ b/historyKeeper.js @@ -5,6 +5,8 @@ const nThen = require('nthen'); const Nacl = require('tweetnacl'); const Crypto = require('crypto'); +const Once = require("./lib/once"); +const Meta = require("./lib/metadata"); let Log; const now = function () { return (new Date()).getTime(); }; @@ -61,51 +63,78 @@ module.exports.create = function (cfg) { const cpIndex = []; let messageBuf = []; let validateKey; - let metadata; // FIXME METADATA + let metadata; // FIXME METADATA READ let i = 0; - store.readMessagesBin(channelName, 0, (msgObj, rmcb) => { - let msg; - i++; - if (!validateKey && msgObj.buff.indexOf('validateKey') > -1) { - metadata = msg = tryParse(msgObj.buff.toString('utf8')); // FIXME METADATA - if (typeof msg === "undefined") { return rmcb(); } - if (msg.validateKey) { - validateKey = historyKeeperKeys[channelName] = msg; - return rmcb(); + + const ref = {}; + + const CB = Once(cb); + + const offsetByHash = {}; + let size = 0; + nThen(function (w) { + store.readMessagesBin(channelName, 0, (msgObj, rmcb) => { + let msg; + i++; + if (!validateKey && msgObj.buff.indexOf('validateKey') > -1) { + metadata = msg = tryParse(msgObj.buff.toString('utf8')); // FIXME METADATA READ + if (typeof msg === "undefined") { return rmcb(); } + if (msg.validateKey) { + validateKey = historyKeeperKeys[channelName] = msg; + return rmcb(); + } } - } - if (msgObj.buff.indexOf('cp|') > -1) { - msg = msg || tryParse(msgObj.buff.toString('utf8')); - if (typeof msg === "undefined") { return rmcb(); } - if (msg[2] === 'MSG' && msg[4].indexOf('cp|') === 0) { - cpIndex.push({ - offset: msgObj.offset, - line: i - }); - messageBuf = []; + if (msgObj.buff.indexOf('cp|') > -1) { + msg = msg || tryParse(msgObj.buff.toString('utf8')); + if (typeof msg === "undefined") { return rmcb(); } + if (msg[2] === 'MSG' && msg[4].indexOf('cp|') === 0) { + cpIndex.push({ + offset: msgObj.offset, + line: i + }); + messageBuf = []; + } + } + messageBuf.push(msgObj); + return rmcb(); + }, w((err) => { + if (err && err.code !== 'ENOENT') { + w.abort(); + return void CB(err); } + messageBuf.forEach((msgObj) => { + const msg = tryParse(msgObj.buff.toString('utf8')); + if (typeof msg === "undefined") { return; } + if (msg[0] === 0 && msg[2] === 'MSG' && typeof(msg[4]) === 'string') { + offsetByHash[getHash(msg[4])] = msgObj.offset; + } + // There is a trailing \n at the end of the file + size = msgObj.offset + msgObj.buff.length + 1; + }); + })); + }).nThen(function (w) { + // get amended metadata + const handler = Meta.createLineHandler(ref, Log.error); + + if (metadata) { + handler(void 0, metadata); } - messageBuf.push(msgObj); - return rmcb(); - }, (err) => { - if (err && err.code !== 'ENOENT') { return void cb(err); } - const offsetByHash = {}; - let size = 0; - messageBuf.forEach((msgObj) => { - const msg = tryParse(msgObj.buff.toString('utf8')); - if (typeof msg === "undefined") { return; } - if (msg[0] === 0 && msg[2] === 'MSG' && typeof(msg[4]) === 'string') { - offsetByHash[getHash(msg[4])] = msgObj.offset; + + store.readDedicatedMetadata(channelName, handler, w(function (err) { + if (err) { + return void Log.error("DEDICATED_METADATA_ERROR", err); } - // There is a trailing \n at the end of the file - size = msgObj.offset + msgObj.buff.length + 1; - }); - cb(null, { + metadata = ref.meta; + })); + }).nThen(function () { + // FIXME METADATA READ + + CB(null, { // Only keep the checkpoints included in the last 100 messages cpIndex: sliceCpIndex(cpIndex, i), offsetByHash: offsetByHash, size: size, - metadata: metadata, // FIXME METADATA + metadata: metadata, // FIXME METADATA STORE line: i }); }); From 5e1257e630debf241a0001f96b5f1fee7646c7e5 Mon Sep 17 00:00:00 2001 From: ClemDee Date: Fri, 5 Jul 2019 10:19:34 +0200 Subject: [PATCH 013/258] Make contextmenu separators hide for submenus too --- www/drive/inner.js | 91 ++++++++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 32 deletions(-) diff --git a/www/drive/inner.js b/www/drive/inner.js index 510eabbe1..b06d56da2 100644 --- a/www/drive/inner.js +++ b/www/drive/inner.js @@ -327,34 +327,56 @@ define([ 'tabindex': '-1', 'data-icon': faColor, }, Messages.fc_color)), -// h('li.dropdown-submenu', [ -// h('a.cp-app-drive-context-test.dropdown-item', { -// 'tabindex': '-1', -// 'data-icon': faFolderOpen, -// }, "TEST"), -// h("ul.dropdown-menu", [ -// h('li', h('a.cp-app-drive-context-subtest1.dropdown-item', { -// 'tabindex': '-1', -// 'data-icon': faFolderOpen, -// }, "Sub test 1")), -// h('li.dropdown-submenu', [ -// h('a.cp-app-drive-context-test.dropdown-item', { -// 'tabindex': '-1', -// 'data-icon': faFolderOpen, -// }, "TEST"), -// h("ul.dropdown-menu", [ -// h('li', h('a.cp-app-drive-context-subtest2.dropdown-item', { -// 'tabindex': '-1', -// 'data-icon': faFolderOpen, -// }, "Sub test 2")), -// h('li', h('a.cp-app-drive-context-subtest3.dropdown-item', { -// 'tabindex': '-1', -// 'data-icon': faFolderOpen, -// }, "Sub test 3")), -// ]), -// ]), -// ]), -// ]), + h('li.dropdown-submenu', [ + h('a.cp-app-drive-context-test.dropdown-item', { + 'tabindex': '-1', + 'data-icon': faFolderOpen, + }, "TEST"), + h("ul.dropdown-menu", [ + h('li', h('a.cp-app-drive-context-subtest1.dropdown-item', { + 'tabindex': '-1', + 'data-icon': faFolderOpen, + }, "Sub test 1")), + h('li.dropdown-submenu', [ + h('a.cp-app-drive-context-test.dropdown-item', { + 'tabindex': '-1', + 'data-icon': faFolderOpen, + }, "TEST"), + h("ul.dropdown-menu", [ + h('li', h('a.cp-app-drive-context-subtest2.dropdown-item', { + 'tabindex': '-1', + 'data-icon': faFolderOpen, + }, "Sub test 2")), + h('li', h('a.cp-app-drive-context-subtest3.dropdown-item', { + 'tabindex': '-1', + 'data-icon': faFolderOpen, + }, "Sub test 3")), + ]), + ]), + $separator.clone()[0], + h('li', h('a.cp-app-drive-context-subtest4.dropdown-item', { + 'tabindex': '-1', + 'data-icon': faFolderOpen, + }, "Sub test 4")), + $separator.clone()[0], + h('li.dropdown-submenu', [ + h('a.cp-app-drive-context-test.dropdown-item', { + 'tabindex': '-1', + 'data-icon': faFolderOpen, + }, "TEST"), + h("ul.dropdown-menu", [ + h('li', h('a.cp-app-drive-context-subtest5.dropdown-item', { + 'tabindex': '-1', + 'data-icon': faFolderOpen, + }, "Sub test 5")), + h('li', h('a.cp-app-drive-context-subtest6.dropdown-item', { + 'tabindex': '-1', + 'data-icon': faFolderOpen, + }, "Sub test 6")), + ]), + ]), + ]), + ]), h('li', h('a.cp-app-drive-context-download.dropdown-item', { 'tabindex': '-1', 'data-icon': faDownload, @@ -1297,12 +1319,11 @@ define([ updateContextButton(); }; - var displayMenu = function (e) { - var $menu = $contextMenu; + // show / hide dropdown separators + var hideSeparators = function ($menu) { var showSep = false; var $lastVisibleSep = null; - // show / hide drop-down divider - $menu.find(".dropdown-menu").children().each(function (i, el) { + $menu.children().each(function (i, el) { var $el = $(el); if ($el.is(".dropdown-divider")) { $el.css("display", showSep ? "list-item" : "none"); @@ -1314,6 +1335,12 @@ define([ } }); if (!showSep && $lastVisibleSep) { $lastVisibleSep.css("display", "none"); } // remove last divider if no options after + } + var displayMenu = function (e) { + var $menu = $contextMenu; + $menu.find(".dropdown-menu").each(function (i, menu) { + hideSeparators($(menu)); + }); // show / hide submenus $menu.find(".dropdown-submenu").each(function (i, el) { var $el = $(el); From b63354532ceeb8f3f170a23e60eb55c93a2f957c Mon Sep 17 00:00:00 2001 From: ClemDee Date: Fri, 5 Jul 2019 12:30:21 +0200 Subject: [PATCH 014/258] Context submenu now respond to click event --- www/drive/inner.js | 45 ++++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/www/drive/inner.js b/www/drive/inner.js index b06d56da2..de78d6da3 100644 --- a/www/drive/inner.js +++ b/www/drive/inner.js @@ -328,7 +328,7 @@ define([ 'data-icon': faColor, }, Messages.fc_color)), h('li.dropdown-submenu', [ - h('a.cp-app-drive-context-test.dropdown-item', { + h('a.cp-app-drive-context-test1.dropdown-item', { 'tabindex': '-1', 'data-icon': faFolderOpen, }, "TEST"), @@ -338,7 +338,7 @@ define([ 'data-icon': faFolderOpen, }, "Sub test 1")), h('li.dropdown-submenu', [ - h('a.cp-app-drive-context-test.dropdown-item', { + h('a.cp-app-drive-context-test2.dropdown-item', { 'tabindex': '-1', 'data-icon': faFolderOpen, }, "TEST"), @@ -360,7 +360,7 @@ define([ }, "Sub test 4")), $separator.clone()[0], h('li.dropdown-submenu', [ - h('a.cp-app-drive-context-test.dropdown-item', { + h('a.cp-app-drive-context-test3.dropdown-item', { 'tabindex': '-1', 'data-icon': faFolderOpen, }, "TEST"), @@ -477,17 +477,34 @@ define([ var $el = $(el); var $a = $el.children().filter("a"); var $sub = $el.find(".dropdown-menu").first(); + var showSubmenu = function () { + $sub.toggleClass("left", $el.offset().left + $el.outerWidth() + $sub.outerWidth() > $(window).width()); + $sub.show(); + }; + var hideSubmenu = function () { + $sub.hide(); + $sub.removeClass("left"); + }; // Add submenu expand icon $a.append(h("span.dropdown-toggle")); // Show / hide submenu $el.hover(function () { - setTimeout(function () { // wait for dom to update - $sub.toggleClass("left", $el.offset().left + $el.outerWidth() + $sub.outerWidth() > $(window).width()); - $sub.show(); - }); + showSubmenu(); }, function () { - $sub.hide(); - $sub.removeClass("left"); + hideSubmenu(); + }); + $el.click(function (e) { + e.stopPropagation(); + if ($el.children().filter(".dropdown-menu:visible").length !== 0) { + console.log("leave", $a[0]); + $el.find(".dropdown-menu").hide(); + hideSubmenu(); + } + else { + console.log("enter", $a[0]); + $el.siblings().find(".dropdown-menu").hide(); + showSubmenu(); + } }); }); return $(menu); @@ -1137,7 +1154,7 @@ define([ show = ['newfolder', 'newsharedfolder', 'newdoc']; break; case 'tree': - show = ['open', 'openro', 'expandall', 'collapseall', 'color', 'download', 'share', 'rename', 'delete', 'deleteowned', 'removesf', 'properties', 'hashtag', 'subtest1', 'subtest2', 'subtest3']; + show = ['open', 'openro', 'expandall', 'collapseall', 'color', 'download', 'share', 'rename', 'delete', 'deleteowned', 'removesf', 'properties', 'hashtag', 'subtest1', 'subtest2', 'subtest3', 'subtest4', 'subtest5', 'subtest6']; break; case 'default': show = ['open', 'openro', 'share', 'openparent', 'delete', 'deleteowned', 'properties', 'hashtag']; @@ -1335,7 +1352,7 @@ define([ } }); if (!showSep && $lastVisibleSep) { $lastVisibleSep.css("display", "none"); } // remove last divider if no options after - } + }; var displayMenu = function (e) { var $menu = $contextMenu; $menu.find(".dropdown-menu").each(function (i, menu) { @@ -1454,7 +1471,7 @@ define([ }); cb(); }; - if (paths.some(function (p) { return manager.comparePath(newPath, p) })) { return void cb(); } + if (paths.some(function (p) { return manager.comparePath(newPath, p); })) { return void cb(); } manager.move(paths, newPath, newCb, copy); }; // Delete paths from the drive and/or shared folders (without moving them to the trash) @@ -3423,6 +3440,7 @@ define([ }; APP.hideMenu = function (e) { + console.error("HIDEMENU", e ? e.target : "e is undefined"); $contextMenu.hide(); $trashTreeContextMenu.hide(); $trashContextMenu.hide(); @@ -3525,7 +3543,7 @@ define([ if (paths.length !== 1) { return; } displayRenameInput(paths[0].element, paths[0].path); } - if ($(this).hasClass("cp-app-drive-context-color")) { + else if ($(this).hasClass("cp-app-drive-context-color")) { var currentColor = getFolderColor(paths[0].path); pickFolderColor(paths[0].element, currentColor, function (color) { paths.forEach(function (p) { @@ -3770,6 +3788,7 @@ define([ $appContainer.on('mouseup', function (e) { //if (sel.down) { return; } if (e.which !== 1) { return ; } + if ($(e.target).is(".dropdown-submenu a, .dropdown-submenu a span")) { return; } // if we click on dropdown-submenu, don't close menu APP.hideMenu(e); //removeSelected(e); }); From 9d476ce9cdd0174116feda3d6d658cb3707b9bff Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 5 Jul 2019 17:06:02 +0200 Subject: [PATCH 015/258] Fix error in the console --- www/common/outer/mailbox.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/common/outer/mailbox.js b/www/common/outer/mailbox.js index 08e7b9351..517bea017 100644 --- a/www/common/outer/mailbox.js +++ b/www/common/outer/mailbox.js @@ -341,8 +341,8 @@ proxy.mailboxes = { var req = ctx.req[txid]; var type = parsed[0]; var _msg = parsed[2]; - var box = req.box; if (!req) { return; } + var box = req.box; if (type === 'HISTORY_RANGE') { if (!Array.isArray(_msg)) { return; } From 9f2355e95b8f686bfa17b9f15ce72d6778d3ff20 Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 10 Jul 2019 13:34:55 +0200 Subject: [PATCH 016/258] Fix scrollbar issue in spreadsheets --- www/common/onlyoffice/app-oo.less | 1 + 1 file changed, 1 insertion(+) diff --git a/www/common/onlyoffice/app-oo.less b/www/common/onlyoffice/app-oo.less index 60510914d..c7bd64e01 100644 --- a/www/common/onlyoffice/app-oo.less +++ b/www/common/onlyoffice/app-oo.less @@ -44,6 +44,7 @@ body.cp-app-sheet, body.cp-app-oodoc, body.cp-app-ooslide { height: 100%; background-color: lightgrey; display: flex; + min-height: 0; } #cp-app-oo-editor { flex: 1; From 44e57619d024b937f7c482b74d65029f9f6bc273 Mon Sep 17 00:00:00 2001 From: Weblate Date: Wed, 10 Jul 2019 12:32:38 +0000 Subject: [PATCH 017/258] Translated using Weblate (Russian) Currently translated at 40.3% (410 of 1017 strings) Translation: CryptPad/App Translate-URL: http://weblate.cryptpad.fr/projects/cryptpad/app/ru/ --- www/common/translations/messages.ru.json | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/www/common/translations/messages.ru.json b/www/common/translations/messages.ru.json index e8f61dfe2..98fe7eb64 100644 --- a/www/common/translations/messages.ru.json +++ b/www/common/translations/messages.ru.json @@ -389,5 +389,27 @@ "fc_empty": "Удалить корзину", "fc_prop": "Свойства", "fc_hashtag": "Теги", - "fc_sizeInKilobytes": "Размер в килобайтах" + "fc_sizeInKilobytes": "Размер в килобайтах", + "poll_title": "Приватный выбор даты", + "fm_moveNestedSF": "Нельзя помещать одну общую папку в другую. Папка {0} не была перемещена.", + "fc_color": "Изменить цвет", + "fc_expandAll": "Расширить все", + "fc_collapseAll": "Скрыть все", + "fc_remove": "Удалить из вашего CryptDrive", + "fo_moveUnsortedError": "Вы не можете переместить папку в список черновиков", + "fo_existingNameError": "Это имя уже используется в данной директории. Пожалуйста выберите другое.", + "fo_unableToRestore": "Невозможно восстановить этот файл в исходное местоположение. Вы можете попытаться переместить его в другое место.", + "login_login": "Войти", + "login_makeAPad": "Создать анонимный пэд", + "login_nologin": "Просмотреть локальные пэды", + "login_register": "Зарегистрироваться", + "logoutButton": "Выйти", + "settingsButton": "Настройки", + "login_username": "Имя пользователя", + "login_password": "Пароль", + "login_confirm": "Подтвердите ваш пароль", + "login_remember": "Запомнить меня", + "login_hashing": "Ваш пароль хэшируется, это может занять некое время.", + "login_hello": "Привет {0},", + "login_helloNoName": "Привет," } From 6f1e281cf877523d31e81defd7082747b0256209 Mon Sep 17 00:00:00 2001 From: yflory Date: Thu, 11 Jul 2019 14:16:04 +0200 Subject: [PATCH 018/258] request capabilities temp --- www/common/cryptpad-common.js | 4 ++ www/common/notifications.js | 27 ++++++++++++- www/common/outer/async-store.js | 60 ++++++++++++++++++++++++++++ www/common/outer/mailbox-handlers.js | 28 +++++++++++++ www/common/outer/mailbox.js | 2 +- www/common/outer/store-rpc.js | 1 + www/common/sframe-app-framework.js | 1 + www/common/sframe-common-outer.js | 7 ++++ www/common/toolbar3.js | 38 ++++++++++++++++++ 9 files changed, 166 insertions(+), 2 deletions(-) diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index fef46951c..bd467b764 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -693,6 +693,10 @@ define([ pad.onConnectEvent = Util.mkEvent(); pad.onErrorEvent = Util.mkEvent(); + pad.requestAccess = function (data, cb) { + postMessage("REQUEST_PAD_ACCESS", data, cb); + }; + common.changePadPassword = function (Crypt, href, newPassword, edPublic, cb) { if (!href) { return void cb({ error: 'EINVAL_HREF' }); } var parsed = Hash.parsePadUrl(href); diff --git a/www/common/notifications.js b/www/common/notifications.js index 2e4bfce62..8aa46fde1 100644 --- a/www/common/notifications.js +++ b/www/common/notifications.js @@ -2,9 +2,10 @@ define([ 'jquery', '/common/hyperscript.js', '/common/common-hash.js', + '/common/common-interface.js', '/common/common-ui-elements.js', '/customize/messages.js', -], function ($, h, Hash, UIElements, Messages) { +], function ($, h, Hash, UI, UIElements, Messages) { var handlers = {}; @@ -84,12 +85,36 @@ define([ key: 'newPadPassword', value: msg.content.password }, todo); + common.mailbox.dismiss(data, function (err) { + if (err) { return void console.error(err); } + }); }; if (!content.archived) { content.dismissHandler = defaultDismiss(common, data); } }; + handlers['REQUEST_PAD_ACCESS'] = function (common, data) { + var content = data.content; + var msg = content.msg; + + // Check authenticity + if (msg.author !== msg.content.user.curvePublic) { return; } + + // Display the notification + content.getFormatText = function () { + return 'Edit access request: ' + msg.content.channel + ' - ' + msg.content.user.displayName; + }; + + // if not archived, add handlers + content.handler = function () { + UI.confirm("Give edit rights?", function (yes) { + if (!yes) { return; } + // XXX Command to worker to get the edit href and send it to msg.content.user + }); + }; + }; + return { add: function (common, data) { var type = data.content.msg.type; diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index fb1bcd202..a4aaf2b5c 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -1090,6 +1090,7 @@ define([ var channels = Store.channels = store.channels = {}; Store.joinPad = function (clientId, data) { + console.log('joining', data.channel); var isNew = typeof channels[data.channel] === "undefined"; var channel = channels[data.channel] = channels[data.channel] || { queue: [], @@ -1243,6 +1244,65 @@ define([ channel.sendMessage(msg, clientId, cb); }; + Store.requestPadAccess = function (clientId, data, cb) { + // Get owners from pad metadata + // Try to find an owner in our friend list + // Mailbox... + var channel = channels[data.channel]; + if (!data.send && channel && (!channel.data || !channel.data.channel)) { + var i = 0; + var it = setInterval(function () { + if (channel.data && channel.data.channel) { + clearInterval(it); + Store.requestPadAccess(clientId, data, cb); + return; + } + if (i >= 300) { // One minute timeout + clearInterval(it); + } + i++ + }, 200); + return; + }; + var fData = channel.data || {}; + if (fData.owners) { + var friends = store.proxy.friends || {}; + if (Object.keys(friends).length > 1) { + var owner; + fData.owners.some(function (edPublic) { + return Object.keys(friends).some(function (curve) { + if (curve === "me") { return; } + if (edPublic === friends[curve].edPublic && + friends[curve].notifications) { + owner = friends[curve]; + return true; + } + }); + }); + if (owner) { + console.log(owner); + if (data.send) { + // XXX send the pad title here...? or get it from the recipient's drive + var myData = Messaging.createData(store.proxy); + delete myData.channel; + store.mailbox.sendTo('REQUEST_PAD_ACCESS', { + channel: data.channel, + user: myData + }, { + channel: owner.notifications, + curvePublic: owner.curvePublic + }, function () { + cb({state: true}); + }); + return; + } + return void cb({state: true}); + } + } + } + cb({sent: false}); + }; + // GET_FULL_HISTORY from sframe-common-outer Store.getFullHistory = function (clientId, data, cb) { var network = store.network; diff --git a/www/common/outer/mailbox-handlers.js b/www/common/outer/mailbox-handlers.js index 10e61fb60..e460233c2 100644 --- a/www/common/outer/mailbox-handlers.js +++ b/www/common/outer/mailbox-handlers.js @@ -201,6 +201,34 @@ define([ } }; + // Incoming edit rights request: add data before sending it to inner + handlers['REQUEST_PAD_ACCESS'] = function (ctx, box, data, hash) { + var msg = data.msg; + var hash = data.hash; + var content = msg.content; + + if (msg.author !== content.user.curvePublic) { return void cb(true); } + + var channel = content.channel; + var res = ctx.store.manager.findChannel(channel); + + if (!res.length) { return void cb(true); } + + var edPublic = store.proxy.edPublic; + var title; + if (!res.some(functon (obj) { + if (obj.data && + Array.isArray(obj.data.owners) && obj.data.owners.indexOf(edPublic) !== -1 && + obj.data.href) { + title = obj.data.filename || obj.data.title; + return true; + } + })) { return void cb(true); } + + content.title = title; + + }); + return { add: function (ctx, box, data, cb) { /** diff --git a/www/common/outer/mailbox.js b/www/common/outer/mailbox.js index 08e7b9351..24d14345f 100644 --- a/www/common/outer/mailbox.js +++ b/www/common/outer/mailbox.js @@ -339,10 +339,10 @@ proxy.mailboxes = { var txid = parsed[1]; var req = ctx.req[txid]; + if (!req) { return; } var type = parsed[0]; var _msg = parsed[2]; var box = req.box; - if (!req) { return; } if (type === 'HISTORY_RANGE') { if (!Array.isArray(_msg)) { return; } diff --git a/www/common/outer/store-rpc.js b/www/common/outer/store-rpc.js index 6b774339c..adfe4115c 100644 --- a/www/common/outer/store-rpc.js +++ b/www/common/outer/store-rpc.js @@ -78,6 +78,7 @@ define([ GET_FULL_HISTORY: Store.getFullHistory, GET_HISTORY_RANGE: Store.getHistoryRange, IS_NEW_CHANNEL: Store.isNewChannel, + REQUEST_PAD_ACCESS: Store.requestPadAccess, // Drive DRIVE_USEROBJECT: Store.userObjectCommand, // Settings, diff --git a/www/common/sframe-app-framework.js b/www/common/sframe-app-framework.js index f1068a6de..af856d1d7 100644 --- a/www/common/sframe-app-framework.js +++ b/www/common/sframe-app-framework.js @@ -603,6 +603,7 @@ define([ 'newpad', 'share', 'limit', + 'request', 'unpinnedWarning', 'notifications' ], diff --git a/www/common/sframe-common-outer.js b/www/common/sframe-common-outer.js index fba08279a..8c6f79eb2 100644 --- a/www/common/sframe-common-outer.js +++ b/www/common/sframe-common-outer.js @@ -941,6 +941,13 @@ define([ sframeChan.event('EV_WORKER_TIMEOUT'); }); + sframeChan.on('Q_REQUEST_ACCESS', function (data, cb) { + Cryptpad.padRpc.requestAccess({ + send: data, + channel: secret.channel + }, cb); + }); + if (cfg.messaging) { Notifier.getPermission(); diff --git a/www/common/toolbar3.js b/www/common/toolbar3.js index d6b116df3..98c2801e7 100644 --- a/www/common/toolbar3.js +++ b/www/common/toolbar3.js @@ -574,6 +574,43 @@ MessengerUI, Messages) { return $shareBlock; }; + var createRequest = function (toolbar, config) { + console.error('test'); + if (!config.metadataMgr) { + throw new Error("You must provide a `metadataMgr` to display the request access button"); + } + + // We can only requets more access if we're in read-only mode + if (config.readOnly !== 1) { return; } + + var $requestBlock = $('