Implement full CryptDrive export

pull/1/head
yflory 6 years ago
parent 92ce311694
commit 3e9e92dcac

@ -0,0 +1,17 @@
// This file is used when a user tries to export the entire CryptDrive.
// Pads from the code app will be exported using this format instead of plain text.
define([
'/common/sframe-common-codemirror.js',
], function (SFCodeMirror) {
var module = {};
module.main = function (userDoc, cb) {
var mode = userDoc.highlightMode || 'gfm';
var content = userDoc.content;
module.type = SFCodeMirror.getContentExtension(mode);
cb(SFCodeMirror.fileExporter(content));
};
return module;
});

@ -12,11 +12,18 @@ define([
S.cb(err, doc);
S.done = true;
var disconnect = Util.find(S, ['network', 'disconnect']);
if (typeof(disconnect) === 'function') { disconnect(); }
var abort = Util.find(S, ['realtime', 'realtime', 'abort']);
if (!S.hasNetwork) {
var disconnect = Util.find(S, ['network', 'disconnect']);
if (typeof(disconnect) === 'function') { disconnect(); }
}
if (S.leave) {
try {
S.leave();
} catch (e) { console.log(e); }
}
var abort = Util.find(S, ['session', 'realtime', 'abort']);
if (typeof(abort) === 'function') {
S.realtime.realtime.sync();
S.session.realtime.sync();
abort();
}
};
@ -51,11 +58,12 @@ define([
opt = opt || {};
var config = makeConfig(hash, opt.password);
var Session = { cb: cb, };
var Session = { cb: cb, hasNetwork: Boolean(opt.network) };
config.onReady = function (info) {
var rt = Session.session = info.realtime;
Session.network = info.network;
Session.leave = info.leave;
finish(Session, void 0, rt.getUserDoc());
};

@ -59,6 +59,19 @@ define([
cb();
};
common.makeNetwork = function (cb) {
require([
'/bower_components/netflux-websocket/netflux-client.js',
'/common/outer/network-config.js'
], function (Netflux, NetConfig) {
var wsUrl = NetConfig.getWebsocketURL();
Netflux.connect(wsUrl).then(function (network) {
cb(null, network);
}, function (err) {
cb(err);
});
});
};
// RESTRICTED
// Settings only

@ -38,6 +38,12 @@ define([
return cursor;
};
module.getContentExtension = function (mode) {
return (Modes.extensionOf(mode) || '.txt').slice(1);
};
module.fileExporter = function (content) {
return new Blob([ content ], { type: 'text/plain;charset=utf-8' });
};
module.setValueAndCursor = function (editor, oldDoc, remoteDoc) {
var scroll = editor.getScrollInfo();
//get old cursor here
@ -271,10 +277,10 @@ define([
};
exp.getContentExtension = function () {
return (Modes.extensionOf(exp.highlightMode) || '.txt').slice(1);
return module.getContentExtension(exp.highlightMode);
};
exp.fileExporter = function () {
return new Blob([ editor.getValue() ], { type: 'text/plain;charset=utf-8' });
return module.fileExporter(editor.getValue());
};
exp.fileImporter = function (content, file) {
var $toolbarContainer = $('#cme_toolbox');

@ -21,7 +21,9 @@ define([
var FilePicker;
var Messaging;
var Notifier;
var Utils = {};
var Utils = {
nThen: nThen
};
var AppConfig;
var Test;
var password;
@ -744,13 +746,46 @@ define([
Cryptpad.removeLoginBlock(data, cb);
});
var cgNetwork;
var whenCGReady = function (cb) {
if (cgNetwork && cgNetwork !== true) { console.log(cgNetwork); return void cb(); }
setTimeout(function () {
whenCGReady(cb);
}, 500);
};
var i = 0;
sframeChan.on('Q_CRYPTGET', function (data, cb) {
Cryptget.get(data.hash, function (err, val) {
cb({
error: err,
data: val
var todo = function () {
data.opts.network = cgNetwork;
Cryptget.get(data.hash, function (err, val) {
cb({
error: err,
data: val
});
}, data.opts);
};
//return void todo();
if (i > 30) {
i = 0;
cgNetwork = undefined;
}
i++
if (!cgNetwork) {
cgNetwork = true;
return void Cryptpad.makeNetwork(function (err, nw) {
console.log(nw);
cgNetwork = nw;
todo();
});
}, data.opts);
} else if (cgNetwork === true) {
return void whenCGReady(todo);
}
todo();
});
sframeChan.on('EV_CRYPTGET_DISCONNECT', function () {
if (!cgNetwork) { return; }
cgNetwork.disconnect();
cgNetwork = undefined;
});
if (cfg.addRpc) {

@ -274,5 +274,6 @@ define({
// Ability to get a pad's content from its hash
'Q_CRYPTGET': true,
'EV_CRYPTGET_DISCONNECT': true,
});

@ -0,0 +1,16 @@
// This file is used when a user tries to export the entire CryptDrive.
// Pads from the code app will be exported using this format instead of plain text.
define([
], function () {
var module = {};
module.main = function (userDoc, cb) {
var content = userDoc.content;
cb(new Blob([JSON.stringify(content, 0, 2)], {
type: 'application/json',
}));
};
return module;
});

@ -368,7 +368,7 @@ define([
}
framework.setFileExporter('json', function () {
return new Blob([JSON.stringify(kanban.getBoardsJSON())], {
return new Blob([JSON.stringify(kanban.getBoardsJSON(), 0, 2)], {
type: 'application/json',
});
});

@ -0,0 +1,64 @@
define([
'jquery',
'/common/common-util.js',
'/bower_components/hyperjson/hyperjson.js',
'/bower_components/nthen/index.js',
], function ($, Util, Hyperjson, nThen) {
var module = {
type: 'html'
};
var exportMediaTags = function (inner, cb) {
var $clone = $(inner).clone();
nThen(function (waitFor) {
$(inner).find('media-tag').each(function (i, el) {
if (!$(el).data('blob') || !el.blob) { return; }
Util.blobToImage(el.blob || $(el).data('blob'), waitFor(function (imgSrc) {
$clone.find('media-tag[src="' + $(el).attr('src') + '"] img')
.attr('src', imgSrc);
$clone.find('media-tag').parent()
.find('.cke_widget_drag_handler_container').remove();
}));
});
}).nThen(function () {
cb($clone[0]);
});
};
module.getHTML = function (inner) {
return ('<!DOCTYPE html>\n' + '<html>\n' +
' <head><meta charset="utf-8"></head>\n <body>' +
inner.innerHTML.replace(/<img[^>]*class="cke_anchor"[^>]*data-cke-realelement="([^"]*)"[^>]*>/g,
function(match,realElt){
//console.log("returning realElt \"" + unescape(realElt)+ "\".");
return decodeURIComponent(realElt); }) +
' </body>\n</html>'
);
};
module.main = function (userDoc, cb) {
var inner;
if (userDoc instanceof Element || userDoc instanceof HTMLElement) {
inner = userDoc;
} else {
try {
if (Array.isArray(userDoc)) {
inner = Hyperjson.toDOM(userDoc);
} else {
console.error('This Pad is not an array!', userDoc);
return void cb('');
}
} catch (e) {
console.log(JSON.stringify(userDoc));
console.error(userDoc);
console.error(e);
return void cb('');
}
}
exportMediaTags(inner, function (toExport) {
cb(new Blob([ module.getHTML(toExport) ], { type: "text/html;charset=utf-8" }));
});
};
return module;
});

@ -25,6 +25,7 @@ define([
'/common/TypingTests.js',
'/customize/messages.js',
'/pad/links.js',
'/pad/export.js',
'/bower_components/nthen/index.js',
'/common/media-tag.js',
'/api/config',
@ -49,6 +50,7 @@ define([
TypingTest,
Messages,
Links,
Exporter,
nThen,
MediaTag,
ApiConfig,
@ -166,17 +168,6 @@ define([
//'AUDIO'
];
var getHTML = function (inner) {
return ('<!DOCTYPE html>\n' + '<html>\n' +
' <head><meta charset="utf-8"></head>\n <body>' +
inner.innerHTML.replace(/<img[^>]*class="cke_anchor"[^>]*data-cke-realelement="([^"]*)"[^>]*>/g,
function(match,realElt){
//console.log("returning realElt \"" + unescape(realElt)+ "\".");
return decodeURIComponent(realElt); }) +
' </body>\n</html>'
);
};
var CKEDITOR_CHECK_INTERVAL = 100;
var ckEditorAvailable = function (cb) {
var intr;
@ -647,26 +638,8 @@ define([
});
}, true);
var exportMediaTags = function (inner, cb) {
var $clone = $(inner).clone();
nThen(function (waitFor) {
$(inner).find('media-tag').each(function (i, el) {
if (!$(el).data('blob') || !el.blob) { return; }
Util.blobToImage(el.blob || $(el).data('blob'), waitFor(function (imgSrc) {
$clone.find('media-tag[src="' + $(el).attr('src') + '"] img')
.attr('src', imgSrc);
$clone.find('media-tag').parent()
.find('.cke_widget_drag_handler_container').remove();
}));
});
}).nThen(function () {
cb($clone[0]);
});
};
framework.setFileExporter('html', function (cb) {
exportMediaTags(inner, function (toExport) {
cb(new Blob([ getHTML(toExport) ], { type: "text/html;charset=utf-8" }));
});
framework.setFileExporter(Exporter.type, function (cb) {
Exporter.main(inner, cb);
}, true);
framework.setNormalizer(function (hjson) {
@ -837,7 +810,7 @@ define([
test.fail("No anchors found. Please adjust document");
} else {
console.log(anchors.length + " anchors found.");
var exported = getHTML(window.inner);
var exported = Exporter.getHTML(window.inner);
console.log("Obtained exported: " + exported);
var allFound = true;
for(var i=0; i<anchors.length; i++) {

@ -0,0 +1,71 @@
// This file is used when a user tries to export the entire CryptDrive.
// Pads from the code app will be exported using this format instead of plain text.
define([
'/customize/messages.js',
], function (Messages) {
var module = {};
var copyObject = function (obj) {
return JSON.parse(JSON.stringify(obj));
};
module.getCSV = function (content) {
if (!APP.proxy) { return; }
var data = copyObject(content);
var res = '';
var escapeStr = function (str) {
return '"' + str.replace(/"/g, '""') + '"';
};
[null].concat(data.rowsOrder).forEach(function (rowId, i) {
[null].concat(data.colsOrder).forEach(function (colId, j) {
// thead
if (i === 0) {
if (j === 0) { res += ','; return; }
if (!colId) { throw new Error("Invalid data"); }
res += escapeStr(data.cols[colId] || Messages.anonymous) + ',';
return;
}
// tbody
if (!rowId) { throw new Error("Invalid data"); }
if (j === 0) {
res += escapeStr(data.rows[rowId] || Messages.poll_optionPlaceholder) + ',';
return;
}
if (!colId) { throw new Error("Invalid data"); }
res += (data.cells[colId + '_' + rowId] || 3) + ',';
});
// last column: total
// thead
if (i === 0) {
res += escapeStr(Messages.poll_total) + '\n';
return;
}
// tbody
if (!rowId) { throw new Error("Invalid data"); }
res += APP.count[rowId] || '?';
res += '\n';
});
return res;
};
module.main = function (userDoc, cb) {
var content = userDoc.content;
var csv;
try {
csv = module.getCSV(content);
} catch (e) {
console.error(e);
var blob2 = new Blob([JSON.stringify(content, 0, 2)], {
type: 'application/json',
});
return void cb(content, true);
}
var blob = new Blob([csv], {type: "application/csv;charset=utf-8"});
cb(blob);
};
return module;
});

@ -9,6 +9,7 @@ define([
'/bower_components/chainpad-listmap/chainpad-listmap.js',
'/customize/pages.js',
'/poll/render.js',
'/poll/export.js',
'/common/diffMarked.js',
'/common/sframe-common-codemirror.js',
'/common/common-thumbnail.js',
@ -38,6 +39,7 @@ define([
Listmap,
Pages,
Renderer,
Exporter,
DiffMd,
SframeCM,
Thumb,
@ -69,55 +71,19 @@ define([
return JSON.parse(JSON.stringify(obj));
};
var getCSV = APP.getCSV = function () {
if (!APP.proxy) { return; }
var data = copyObject(APP.proxy.content);
var res = '';
var escapeStr = function (str) {
return '"' + str.replace(/"/g, '""') + '"';
};
[null].concat(data.rowsOrder).forEach(function (rowId, i) {
[null].concat(data.colsOrder).forEach(function (colId, j) {
// thead
if (i === 0) {
if (j === 0) { res += ','; return; }
if (!colId) { throw new Error("Invalid data"); }
res += escapeStr(data.cols[colId] || Messages.anonymous) + ',';
return;
}
// tbody
if (!rowId) { throw new Error("Invalid data"); }
if (j === 0) {
res += escapeStr(data.rows[rowId] || Messages.poll_optionPlaceholder) + ',';
return;
}
if (!colId) { throw new Error("Invalid data"); }
res += (data.cells[colId + '_' + rowId] || 3) + ',';
});
// last column: total
// thead
if (i === 0) {
res += escapeStr(Messages.poll_total) + '\n';
return;
}
// tbody
if (!rowId) { throw new Error("Invalid data"); }
res += APP.count[rowId] || '?';
res += '\n';
});
return res;
APP.getCSV = function () {
return Exporter.getCSV(APP.proxy.content);
};
var exportFile = function () {
var csv = getCSV();
var suggestion = Title.suggestTitle(Title.defaultTitle);
UI.prompt(Messages.exportPrompt,
Util.fixFileName(suggestion) + '.csv', function (filename) {
if (!(typeof(filename) === 'string' && filename)) { return; }
var blob = new Blob([csv], {type: "application/csv;charset=utf-8"});
saveAs(blob, filename);
Exporter.main(APP.proxy, function (blob, isJson) {
var suggestion = Title.suggestTitle(Title.defaultTitle);
var ext = isJson ? '.json' : '.csv';
UI.prompt(Messages.exportPrompt,
Util.fixFileName(suggestion) + ext, function (filename) {
if (!(typeof(filename) === 'string' && filename)) { return; }
saveAs(blob, filename);
});
});
};

@ -862,13 +862,13 @@ define([
var accountName = privateData.accountName;
var displayName = metadataMgr.getUserData().name || '';
var name = displayName || accountName || Messages.anonymous;
var suggestion = name + '-' + new Date().toDateString();
var exportFile = function () {
sframeChan.query("Q_SETTINGS_DRIVE_GET", null, function (err, data) {
if (err) { return void console.error(err); }
var sjson = JSON.stringify(data);
var name = displayName || accountName || Messages.anonymous;
var suggestion = name + '-' + new Date().toDateString();
UI.prompt(Messages.exportPrompt,
Util.fixFileName(suggestion) + '.json', function (filename) {
if (!(typeof(filename) === 'string' && filename)) { return; }
@ -916,16 +916,14 @@ define([
Backup.create(data, getPad, function (blob) {
saveAs(blob, filename);
sframeChan.event('EV_CRYPTGET_DISCONNECT');
});
};
sframeChan.query("Q_SETTINGS_DRIVE_GET", null, function (err, data) {
sframeChan.query("Q_SETTINGS_DRIVE_GET", "full", function (err, data) {
if (err) { return void console.error(err); }
var sjson = JSON.stringify(data);
var name = displayName || accountName || Messages.anonymous;
var suggestion = name + '-' + new Date().toDateString();
UI.prompt('TODO are you sure? if ye,s pick a name...', // XXX
Util.fixFileName(suggestion) + '.json', function (filename) {
if (data.error) { return void console.error(data.error); }
UI.prompt('TODO are you sure? if yes, pick a name...', // XXX
Util.fixFileName(suggestion) + '.zip', function (filename) {
if (!(typeof(filename) === 'string' && filename)) { return; }
todo(data, filename);
});

@ -43,7 +43,26 @@ define([
});
});
sframeChan.on('Q_SETTINGS_DRIVE_GET', function (d, cb) {
Cryptpad.getUserObject(cb);
if (d === "full") {
// We want shared folders too
}
Cryptpad.getUserObject(function (obj) {
if (obj.error) { return void cb(obj); }
var result = {
uo: obj,
sf: {}
};
if (!obj.drive || !obj.drive.sharedFolders) { return void cb(result); }
Utils.nThen(function (waitFor) {
Object.keys(obj.drive.sharedFolders).forEach(function (id) {
Cryptpad.getSharedFolder(id, waitFor(function (obj) {
result.sf[id] = obj;
}));
});
}).nThen(function () {
cb(result);
});
});
});
sframeChan.on('Q_SETTINGS_DRIVE_SET', function (data, cb) {
var sjson = JSON.stringify(data);

@ -1,24 +1,51 @@
define([
'/common/cryptget.js',
'/common/common-hash.js',
'/common/common-util.js',
'/file/file-crypto.js',
'/bower_components/nthen/index.js',
'/bower_components/saferphore/index.js',
'/bower_components/jszip/dist/jszip.min.js',
], function (Crypt, Hash, nThen, Saferphore, JsZip) {
], function (Crypt, Hash, Util, FileCrypto, nThen, Saferphore, JsZip) {
var sanitize = function (str) {
return str.replace(/[^a-z0-9]/gi, '_').toLowerCase();
return str.replace(/[\\/?%*:|"<>]/gi, '_')/*.toLowerCase()*/;
};
var getUnique = function (name, ext, existing) {
var n = name;
var n = name + ext;
var i = 1;
while (existing.indexOf(n) !== -1) {
n = name + ' ('+ i++ + ')';
while (existing.indexOf(n.toLowerCase()) !== -1) {
n = name + ' ('+ i++ + ')' + ext;
}
return n;
};
var transform = function (ctx, type, sjson, cb) {
var result = {
data: sjson,
ext: '.json',
};
var json;
try {
json = JSON.parse(sjson);
} catch (e) {
return void cb(result);
}
var path = '/' + type + '/export.js';
require([path], function (Exporter) {
Exporter.main(json, function (data) {
result.ext = '.' + Exporter.type;
result.data = data;
cb(result);
});
}, function () {
cb(result);
});
};
// Add a file to the zip. We have to cryptget&transform it if it's a pad
// or fetch&decrypt it if it's a file.
var addFile = function (ctx, zip, fData, existingNames) {
if (!fData.href && !fData.roHref) {
return void ctx.errors.push({
@ -28,70 +55,121 @@ define([
}
var parsed = Hash.parsePadUrl(fData.href || fData.roHref);
// TODO deal with files here
if (parsed.hashData.type !== 'pad') { return; }
if (['pad', 'file'].indexOf(parsed.hashData.type) === -1) { return; }
// waitFor is used to make sure all the pads and files are process before downloading the zip.
var w = ctx.waitFor();
// Work with only 10 pad/files at a time
ctx.sem.take(function (give) {
var opts = {
password: fData.password
};
var rawName = fData.fileName || fData.title || 'File';
var rawName = fData.filename || fData.title || 'File';
console.log(rawName);
ctx.get({
hash: parsed.hash,
opts: opts
}, give(function (err, val) {
var g = give();
var done = function () {
//setTimeout(g, 2000);
g();
w();
if (err) {
return void ctx.errors.push({
error: err,
data: fData
};
var error = function (err) {
done();
return void ctx.errors.push({
error: err,
data: fData
});
};
// Pads (pad,code,slide,kanban,poll,...)
var todoPad = function () {
ctx.get({
hash: parsed.hash,
opts: opts
}, function (err, val) {
if (err) { return void error(err); }
if (!val) { return void error('EEMPTY'); }
var opts = {
binary: true,
};
transform(ctx, parsed.type, val, function (res) {
if (!res.data) { return void error('EEMPTY'); }
var fileName = getUnique(sanitize(rawName), res.ext, existingNames);
existingNames.push(fileName.toLowerCase());
zip.file(fileName, res.data, opts);
console.log('DONE ---- ' + fileName);
setTimeout(done, 1000);
});
}
// TODO transform file here
// var blob = transform(val, type);
var opts = {};
var fileName = getUnique(sanitize(rawName), '.txt', existingNames);
existingNames.push(fileName);
zip.file(fileName, val, opts);
console.log('DONE ---- ' + rawName);
}));
});
};
// Files (mediatags...)
var todoFile = function () {
var secret = Hash.getSecrets('file', parsed.hash, fData.password);
var hexFileName = secret.channel;
var src = Hash.getBlobPathFromHex(hexFileName);
var key = secret.keys && secret.keys.cryptKey;
Util.fetch(src, function (err, u8) {
if (err) { return void error('E404'); }
FileCrypto.decrypt(u8, key, function (err, res) {
if (err) { return void error(err); }
var opts = {
binary: true,
};
var extIdx = rawName.lastIndexOf('.');
var name = extIdx !== -1 ? rawName.slice(0,extIdx) : rawName;
var ext = extIdx !== -1 ? rawName.slice(extIdx) : "";
var fileName = getUnique(sanitize(name), ext, existingNames);
existingNames.push(fileName.toLowerCase());
zip.file(fileName, res.content, opts);
console.log('DONE ---- ' + fileName);
setTimeout(done, 1000);
});
});
};
if (parsed.hashData.type === 'file') {
return void todoFile();
}
todoPad();
});
// cb(err, blob);
// wiht blob.name not undefined
};
var makeFolder = function (ctx, root, zip) {
// Add folders and their content recursively in the zip
var makeFolder = function (ctx, root, zip, fd) {
if (typeof (root) !== "object") { return; }
var existingNames = [];
Object.keys(root).forEach(function (k) {
var el = root[k];
if (typeof el === "object") {
var fName = getUnique(sanitize(k), '', existingNames);
existingNames.push(fName);
return void makeFolder(ctx, el, zip.folder(fName));
existingNames.push(fName.toLowerCase());
return void makeFolder(ctx, el, zip.folder(fName), fd);
}
if (ctx.data.sharedFolders[el]) {
// TODO later...
return;
var sfData = ctx.sf[el].metadata;
var sfName = getUnique(sanitize(sfData.title || 'Folder'), '', existingNames);
existingNames.push(sfName.toLowerCase());
return void makeFolder(ctx, ctx.sf[el].root, zip.folder(sfName), ctx.sf[el].filesData);
}
var fData = ctx.data.filesData[el];
var fData = fd[el];
if (fData) {
addFile(ctx, zip, fData, existingNames);
return;
}
// What is this element?
console.error(el);
});
};
// Main function. Create the empty zip and fill it starting from drive.root
var create = function (data, getPad, cb) {
if (!data || !data.drive) { return void cb('EEMPTY'); }
var sem = Saferphore.create(10);
if (!data || !data.uo || !data.uo.drive) { return void cb('EEMPTY'); }
var sem = Saferphore.create(5);
var ctx = {
get: getPad,
data: data.drive,
data: data.uo.drive,
sf: data.sf,
zip: new JsZip(),
errors: [],
sem: sem,
@ -99,9 +177,8 @@ define([
nThen(function (waitFor) {
ctx.waitFor = waitFor;
var zipRoot = ctx.zip.folder('Root');
makeFolder(ctx, data.drive.root, zipRoot);
makeFolder(ctx, ctx.data.root, zipRoot, ctx.data.filesData);
}).nThen(function () {
// TODO call cb with ctx.zip here
console.log(ctx.zip);
console.log(ctx.errors);
ctx.zip.generateAsync({type: 'blob'}).then(function (content) {

@ -0,0 +1,18 @@
// This file is used when a user tries to export the entire CryptDrive.
// Pads from the slide app will be exported using this format instead of plain text.
define([
'/common/sframe-common-codemirror.js',
], function (SFCodeMirror) {
var module = {
type: 'md'
};
module.main = function (userDoc, cb) {
var content = userDoc.content;
cb(SFCodeMirror.fileExporter(content));
};
return module;
});

@ -0,0 +1,24 @@
// This file is used when a user tries to export the entire CryptDrive.
// Pads from the code app will be exported using this format instead of plain text.
define([
'/bower_components/secure-fabric.js/dist/fabric.min.js',
], function () {
var module = {};
var Fabric = window.fabric;
module.main = function (userDoc, cb) {
var canvas_node = document.createElement('canvas');
canvas_node.setAttribute('style', 'width:600px;height:600px;');
canvas_node.setAttribute('width', '600');
canvas_node.setAttribute('height', '600');
var canvas = new Fabric.Canvas(canvas_node);
var content = userDoc.content;
canvas.loadFromJSON(content, function () {
module.type = 'svg';
cb(canvas.toSVG());
});
};
return module;
});

@ -277,6 +277,7 @@ define([
// Start of the main loop
var andThen2 = function (framework) {
APP.framework = framework;
var canvas = APP.canvas = new Fabric.Canvas('cp-app-whiteboard-canvas', {
containerClass: 'cp-app-whiteboard-canvas-container'
});

Loading…
Cancel
Save