From 51c5b7d3e44ec1ca22799741eb0cdcdcbd670bc6 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 15 Jun 2021 14:48:58 +0200 Subject: [PATCH 01/79] Kanban import from Trello --- www/kanban/export.js | 67 ++++++++++++++++++++++++++++++++++++++++++++ www/kanban/inner.js | 9 +++++- 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/www/kanban/export.js b/www/kanban/export.js index 9ee770ac7..694724cf4 100644 --- a/www/kanban/export.js +++ b/www/kanban/export.js @@ -13,6 +13,73 @@ define([ })); }; + module.import = function (content) { + // Import from Trello + + var c = { + data: {}, + items: {}, + list: [] + }; + + var colorMap = { + red: 'color1', + orange: 'color2', + yellow: 'color3', + lime: 'color4', + green: 'color5', + sky: 'color6', + blue: 'color7', + purple: 'color8', + pink: 'color9', + black: 'nocolor' + }; + content.cards.forEach(function (obj, i) { + var tags; + var color; + if (Array.isArray(obj.labels)) { + obj.labels.forEach(function (l) { + if (!color) { + color = colorMap[l.color] || ''; + } + if (l.name) { + tags = tags || []; + var n = l.name.toLowerCase().trim(); + if (tags.indexOf(n) === -1) { tags.push(n); } + } + }); + } + c.items[(i+1)] = { + id: (i+1), + title: obj.name, + body: obj.desc, + color: color, + tags: tags + }; + }); + + var id = 1; + content.lists.forEach(function (obj) { + var _id = obj.id; + var cards = []; + content.cards.forEach(function (card, i) { + if (card.idList === _id) { + cards.push(i+1); + } + }); + c.data[id] = { + id: id, + title: obj.name, + item: cards + }; + c.list.push(id); + + id++; + }); + + return c; + }; + return module; }); diff --git a/www/kanban/inner.js b/www/kanban/inner.js index 76a9fe1cc..8eb090ffc 100644 --- a/www/kanban/inner.js +++ b/www/kanban/inner.js @@ -18,6 +18,7 @@ define([ '/bower_components/marked/marked.min.js', 'cm/lib/codemirror', '/kanban/jkanban_cp.js', + '/kanban/export.js', 'cm/mode/gfm/gfm', 'cm/addon/edit/closebrackets', @@ -50,7 +51,8 @@ define([ ChainPad, Marked, CodeMirror, - jKanban) + jKanban, + Export) { var verbose = function (x) { console.log(x); }; @@ -1060,6 +1062,11 @@ define([ var parsed; try { parsed = JSON.parse(content); } catch (e) { return void console.error(e); } + + if (parsed && parsed.id && parsed.lists && parsed.cards) { + return { content: Export.import(parsed) }; + } + return { content: parsed }; }); From 6f9127221763d517f0d87ed216f8bb2429945d85 Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 16 Jun 2021 11:17:00 +0200 Subject: [PATCH 02/79] Add default form templates --- www/form/templates.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 www/form/templates.js diff --git a/www/form/templates.js b/www/form/templates.js new file mode 100644 index 000000000..6ff3249b8 --- /dev/null +++ b/www/form/templates.js @@ -0,0 +1,20 @@ +define([ + '/customize/messages.js' +], function (Messages) { + return [{ + id: 'a', + used: 1, + name: Messages.form_type_poll, + content: { + form: { + "1": { + type: 'md' + }, + "2": { + type: 'poll' + } + }, + order: ["1", "2"] + } + }]; +}); From 3f5e3a14afa16c6b820714367caaf636118ff5d9 Mon Sep 17 00:00:00 2001 From: yflory Date: Thu, 17 Jun 2021 18:05:25 +0200 Subject: [PATCH 03/79] Fix allDay events on Safari --- www/calendar/inner.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/www/calendar/inner.js b/www/calendar/inner.js index 58ad22ac1..21cb12789 100644 --- a/www/calendar/inner.js +++ b/www/calendar/inner.js @@ -17,6 +17,7 @@ define([ '/customize/application_config.js', '/lib/calendar/tui-calendar.min.js', '/calendar/export.js', + '/lib/datepicker/flatpickr.js', '/common/inner/share.js', '/common/inner/access.js', @@ -46,6 +47,7 @@ define([ AppConfig, Calendar, Export, + Flatpickr, Share, Access, Properties ) { @@ -169,9 +171,9 @@ define([ var obj = data.content[uid]; obj.title = obj.title || ""; obj.location = obj.location || ""; - if (obj.isAllDay && obj.startDay) { obj.start = +new Date(obj.startDay); } + if (obj.isAllDay && obj.startDay) { obj.start = +Flatpickr.parseDate((obj.startDay)); } if (obj.isAllDay && obj.endDay) { - var endDate = new Date(obj.endDay); + var endDate = Flatpickr.parseDate(obj.endDay); endDate.setHours(23); endDate.setMinutes(59); endDate.setSeconds(59); From a482d6255328c5e1639ed5a4016747e600000a7e Mon Sep 17 00:00:00 2001 From: yflory Date: Thu, 17 Jun 2021 18:12:21 +0200 Subject: [PATCH 04/79] Fix allDay event reminders on Safari --- www/common/notifications.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/www/common/notifications.js b/www/common/notifications.js index e6cad99e2..20738e237 100644 --- a/www/common/notifications.js +++ b/www/common/notifications.js @@ -8,7 +8,8 @@ define([ '/common/common-constants.js', '/customize/messages.js', '/customize/pages.js', -], function($, h, Hash, UI, UIElements, Util, Constants, Messages, Pages) { + '/lib/datepicker/flatpickr.js', +], function($, h, Hash, UI, UIElements, Util, Constants, Messages, Pages, Flatpickr) { var handlers = {}; @@ -477,7 +478,7 @@ define([ var nowDateStr = new Date().toLocaleDateString(); var startDate = new Date(start); if (msg.isAllDay && msg.startDay) { - startDate = new Date(msg.startDay); + startDate = Flatpickr.parseDate(msg.startDay); } // Missed events From a4bd4e27845986c082c0e8176aca031864024f78 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 22 Jun 2021 12:04:56 +0200 Subject: [PATCH 05/79] Support Markdown import/export in Rich text pads --- www/common/diffMarked.js | 12 +- www/lib/turndown.browser.umd.js | 959 ++++++++++++++++++++++++++++++++ www/pad/export.js | 25 +- www/pad/inner.js | 9 +- 4 files changed, 1000 insertions(+), 5 deletions(-) create mode 100644 www/lib/turndown.browser.umd.js diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index cb95f37aa..4bc5c7c5f 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -172,12 +172,16 @@ define([ return h('div.cp-md-toc', content).outerHTML; }; - DiffMd.render = function (md, sanitize, restrictedMd) { + var noHeadingId = false; + DiffMd.render = function (md, sanitize, restrictedMd, noId) { Marked.setOptions({ renderer: restrictedMd ? restrictedRenderer : renderer, }); + noHeadingId = noId; var r = Marked(md, { - sanitize: sanitize + sanitize: sanitize, + headerIds: !noId, + gfm: true, }); // Add Table of Content @@ -207,7 +211,11 @@ define([ }; restrictedRenderer.code = renderer.code; + var _heading = renderer.heading; renderer.heading = function (text, level) { + if (noHeadingId) { + return _heading.apply(this, arguments); + } var i = 0; var safeText = text.toLowerCase().replace(/[^\w]+/g, '-'); var getId = function () { diff --git a/www/lib/turndown.browser.umd.js b/www/lib/turndown.browser.umd.js new file mode 100644 index 000000000..9b8c40d13 --- /dev/null +++ b/www/lib/turndown.browser.umd.js @@ -0,0 +1,959 @@ +(function (global, factory) { + typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() : + typeof define === 'function' && define.amd ? define(factory) : + (global = global || self, global.TurndownService = factory()); +}(this, (function () { 'use strict'; + + function extend (destination) { + for (var i = 1; i < arguments.length; i++) { + var source = arguments[i]; + for (var key in source) { + if (source.hasOwnProperty(key)) destination[key] = source[key]; + } + } + return destination + } + + function repeat (character, count) { + return Array(count + 1).join(character) + } + + var blockElements = [ + 'ADDRESS', 'ARTICLE', 'ASIDE', 'AUDIO', 'BLOCKQUOTE', 'BODY', 'CANVAS', + 'CENTER', 'DD', 'DIR', 'DIV', 'DL', 'DT', 'FIELDSET', 'FIGCAPTION', 'FIGURE', + 'FOOTER', 'FORM', 'FRAMESET', 'H1', 'H2', 'H3', 'H4', 'H5', 'H6', 'HEADER', + 'HGROUP', 'HR', 'HTML', 'ISINDEX', 'LI', 'MAIN', 'MENU', 'NAV', 'NOFRAMES', + 'NOSCRIPT', 'OL', 'OUTPUT', 'P', 'PRE', 'SECTION', 'TABLE', 'TBODY', 'TD', + 'TFOOT', 'TH', 'THEAD', 'TR', 'UL' + ]; + + function isBlock (node) { + return is(node, blockElements) + } + + var voidElements = [ + 'AREA', 'BASE', 'BR', 'COL', 'COMMAND', 'EMBED', 'HR', 'IMG', 'INPUT', + 'KEYGEN', 'LINK', 'META', 'PARAM', 'SOURCE', 'TRACK', 'WBR' + ]; + + function isVoid (node) { + return is(node, voidElements) + } + + function hasVoid (node) { + return has(node, voidElements) + } + + var meaningfulWhenBlankElements = [ + 'A', 'TABLE', 'THEAD', 'TBODY', 'TFOOT', 'TH', 'TD', 'IFRAME', 'SCRIPT', + 'AUDIO', 'VIDEO' + ]; + + function isMeaningfulWhenBlank (node) { + return is(node, meaningfulWhenBlankElements) + } + + function hasMeaningfulWhenBlank (node) { + return has(node, meaningfulWhenBlankElements) + } + + function is (node, tagNames) { + return tagNames.indexOf(node.nodeName) >= 0 + } + + function has (node, tagNames) { + return ( + node.getElementsByTagName && + tagNames.some(function (tagName) { + return node.getElementsByTagName(tagName).length + }) + ) + } + + var rules = {}; + + rules.paragraph = { + filter: 'p', + + replacement: function (content) { + return '\n\n' + content + '\n\n' + } + }; + + rules.lineBreak = { + filter: 'br', + + replacement: function (content, node, options) { + return options.br + '\n' + } + }; + + rules.heading = { + filter: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'], + + replacement: function (content, node, options) { + var hLevel = Number(node.nodeName.charAt(1)); + + if (options.headingStyle === 'setext' && hLevel < 3) { + var underline = repeat((hLevel === 1 ? '=' : '-'), content.length); + return ( + '\n\n' + content + '\n' + underline + '\n\n' + ) + } else { + return '\n\n' + repeat('#', hLevel) + ' ' + content + '\n\n' + } + } + }; + + rules.blockquote = { + filter: 'blockquote', + + replacement: function (content) { + content = content.replace(/^\n+|\n+$/g, ''); + content = content.replace(/^/gm, '> '); + return '\n\n' + content + '\n\n' + } + }; + + rules.list = { + filter: ['ul', 'ol'], + + replacement: function (content, node) { + var parent = node.parentNode; + if (parent.nodeName === 'LI' && parent.lastElementChild === node) { + return '\n' + content + } else { + return '\n\n' + content + '\n\n' + } + } + }; + + rules.listItem = { + filter: 'li', + + replacement: function (content, node, options) { + content = content + .replace(/^\n+/, '') // remove leading newlines + .replace(/\n+$/, '\n') // replace trailing newlines with just a single one + .replace(/\n/gm, '\n '); // indent + var prefix = options.bulletListMarker + ' '; + var parent = node.parentNode; + if (parent.nodeName === 'OL') { + var start = parent.getAttribute('start'); + var index = Array.prototype.indexOf.call(parent.children, node); + prefix = (start ? Number(start) + index : index + 1) + '. '; + } + return ( + prefix + content + (node.nextSibling && !/\n$/.test(content) ? '\n' : '') + ) + } + }; + + rules.indentedCodeBlock = { + filter: function (node, options) { + return ( + options.codeBlockStyle === 'indented' && + node.nodeName === 'PRE' && + node.firstChild && + node.firstChild.nodeName === 'CODE' + ) + }, + + replacement: function (content, node, options) { + return ( + '\n\n ' + + node.firstChild.textContent.replace(/\n/g, '\n ') + + '\n\n' + ) + } + }; + + rules.fencedCodeBlock = { + filter: function (node, options) { + return ( + options.codeBlockStyle === 'fenced' && + node.nodeName === 'PRE' && + node.firstChild && + node.firstChild.nodeName === 'CODE' + ) + }, + + replacement: function (content, node, options) { + var className = node.firstChild.getAttribute('class') || ''; + var language = (className.match(/language-(\S+)/) || [null, ''])[1]; + var code = node.firstChild.textContent; + + var fenceChar = options.fence.charAt(0); + var fenceSize = 3; + var fenceInCodeRegex = new RegExp('^' + fenceChar + '{3,}', 'gm'); + + var match; + while ((match = fenceInCodeRegex.exec(code))) { + if (match[0].length >= fenceSize) { + fenceSize = match[0].length + 1; + } + } + + var fence = repeat(fenceChar, fenceSize); + + return ( + '\n\n' + fence + language + '\n' + + code.replace(/\n$/, '') + + '\n' + fence + '\n\n' + ) + } + }; + + rules.horizontalRule = { + filter: 'hr', + + replacement: function (content, node, options) { + return '\n\n' + options.hr + '\n\n' + } + }; + + rules.inlineLink = { + filter: function (node, options) { + return ( + options.linkStyle === 'inlined' && + node.nodeName === 'A' && + node.getAttribute('href') + ) + }, + + replacement: function (content, node) { + var href = node.getAttribute('href'); + var title = cleanAttribute(node.getAttribute('title')); + if (title) title = ' "' + title + '"'; + return '[' + content + '](' + href + title + ')' + } + }; + + rules.referenceLink = { + filter: function (node, options) { + return ( + options.linkStyle === 'referenced' && + node.nodeName === 'A' && + node.getAttribute('href') + ) + }, + + replacement: function (content, node, options) { + var href = node.getAttribute('href'); + var title = cleanAttribute(node.getAttribute('title')); + if (title) title = ' "' + title + '"'; + var replacement; + var reference; + + switch (options.linkReferenceStyle) { + case 'collapsed': + replacement = '[' + content + '][]'; + reference = '[' + content + ']: ' + href + title; + break + case 'shortcut': + replacement = '[' + content + ']'; + reference = '[' + content + ']: ' + href + title; + break + default: + var id = this.references.length + 1; + replacement = '[' + content + '][' + id + ']'; + reference = '[' + id + ']: ' + href + title; + } + + this.references.push(reference); + return replacement + }, + + references: [], + + append: function (options) { + var references = ''; + if (this.references.length) { + references = '\n\n' + this.references.join('\n') + '\n\n'; + this.references = []; // Reset references + } + return references + } + }; + + rules.emphasis = { + filter: ['em', 'i'], + + replacement: function (content, node, options) { + if (!content.trim()) return '' + return options.emDelimiter + content + options.emDelimiter + } + }; + + rules.strong = { + filter: ['strong', 'b'], + + replacement: function (content, node, options) { + if (!content.trim()) return '' + return options.strongDelimiter + content + options.strongDelimiter + } + }; + + rules.code = { + filter: function (node) { + var hasSiblings = node.previousSibling || node.nextSibling; + var isCodeBlock = node.parentNode.nodeName === 'PRE' && !hasSiblings; + + return node.nodeName === 'CODE' && !isCodeBlock + }, + + replacement: function (content) { + if (!content.trim()) return '' + + var delimiter = '`'; + var leadingSpace = ''; + var trailingSpace = ''; + var matches = content.match(/`+/gm); + if (matches) { + if (/^`/.test(content)) leadingSpace = ' '; + if (/`$/.test(content)) trailingSpace = ' '; + while (matches.indexOf(delimiter) !== -1) delimiter = delimiter + '`'; + } + + return delimiter + leadingSpace + content + trailingSpace + delimiter + } + }; + + rules.image = { + filter: 'img', + + replacement: function (content, node) { + var alt = cleanAttribute(node.getAttribute('alt')); + var src = node.getAttribute('src') || ''; + var title = cleanAttribute(node.getAttribute('title')); + var titlePart = title ? ' "' + title + '"' : ''; + return src ? '![' + alt + ']' + '(' + src + titlePart + ')' : '' + } + }; + + function cleanAttribute (attribute) { + return attribute ? attribute.replace(/(\n+\s*)+/g, '\n') : '' + } + + /** + * Manages a collection of rules used to convert HTML to Markdown + */ + + function Rules (options) { + this.options = options; + this._keep = []; + this._remove = []; + + this.blankRule = { + replacement: options.blankReplacement + }; + + this.keepReplacement = options.keepReplacement; + + this.defaultRule = { + replacement: options.defaultReplacement + }; + + this.array = []; + for (var key in options.rules) this.array.push(options.rules[key]); + } + + Rules.prototype = { + add: function (key, rule) { + this.array.unshift(rule); + }, + + keep: function (filter) { + this._keep.unshift({ + filter: filter, + replacement: this.keepReplacement + }); + }, + + remove: function (filter) { + this._remove.unshift({ + filter: filter, + replacement: function () { + return '' + } + }); + }, + + forNode: function (node) { + if (node.isBlank) return this.blankRule + var rule; + + if ((rule = findRule(this.array, node, this.options))) return rule + if ((rule = findRule(this._keep, node, this.options))) return rule + if ((rule = findRule(this._remove, node, this.options))) return rule + + return this.defaultRule + }, + + forEach: function (fn) { + for (var i = 0; i < this.array.length; i++) fn(this.array[i], i); + } + }; + + function findRule (rules, node, options) { + for (var i = 0; i < rules.length; i++) { + var rule = rules[i]; + if (filterValue(rule, node, options)) return rule + } + return void 0 + } + + function filterValue (rule, node, options) { + var filter = rule.filter; + if (typeof filter === 'string') { + if (filter === node.nodeName.toLowerCase()) return true + } else if (Array.isArray(filter)) { + if (filter.indexOf(node.nodeName.toLowerCase()) > -1) return true + } else if (typeof filter === 'function') { + if (filter.call(rule, node, options)) return true + } else { + throw new TypeError('`filter` needs to be a string, array, or function') + } + } + + /** + * The collapseWhitespace function is adapted from collapse-whitespace + * by Luc Thevenard. + * + * The MIT License (MIT) + * + * Copyright (c) 2014 Luc Thevenard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + + /** + * collapseWhitespace(options) removes extraneous whitespace from an the given element. + * + * @param {Object} options + */ + function collapseWhitespace (options) { + var element = options.element; + var isBlock = options.isBlock; + var isVoid = options.isVoid; + var isPre = options.isPre || function (node) { + return node.nodeName === 'PRE' + }; + + if (!element.firstChild || isPre(element)) return + + var prevText = null; + var prevVoid = false; + + var prev = null; + var node = next(prev, element, isPre); + + while (node !== element) { + if (node.nodeType === 3 || node.nodeType === 4) { // Node.TEXT_NODE or Node.CDATA_SECTION_NODE + var text = node.data.replace(/[ \r\n\t]+/g, ' '); + + if ((!prevText || / $/.test(prevText.data)) && + !prevVoid && text[0] === ' ') { + text = text.substr(1); + } + + // `text` might be empty at this point. + if (!text) { + node = remove(node); + continue + } + + node.data = text; + + prevText = node; + } else if (node.nodeType === 1) { // Node.ELEMENT_NODE + if (isBlock(node) || node.nodeName === 'BR') { + if (prevText) { + prevText.data = prevText.data.replace(/ $/, ''); + } + + prevText = null; + prevVoid = false; + } else if (isVoid(node)) { + // Avoid trimming space around non-block, non-BR void elements. + prevText = null; + prevVoid = true; + } + } else { + node = remove(node); + continue + } + + var nextNode = next(prev, node, isPre); + prev = node; + node = nextNode; + } + + if (prevText) { + prevText.data = prevText.data.replace(/ $/, ''); + if (!prevText.data) { + remove(prevText); + } + } + } + + /** + * remove(node) removes the given node from the DOM and returns the + * next node in the sequence. + * + * @param {Node} node + * @return {Node} node + */ + function remove (node) { + var next = node.nextSibling || node.parentNode; + + node.parentNode.removeChild(node); + + return next + } + + /** + * next(prev, current, isPre) returns the next node in the sequence, given the + * current and previous nodes. + * + * @param {Node} prev + * @param {Node} current + * @param {Function} isPre + * @return {Node} + */ + function next (prev, current, isPre) { + if ((prev && prev.parentNode === current) || isPre(current)) { + return current.nextSibling || current.parentNode + } + + return current.firstChild || current.nextSibling || current.parentNode + } + + /* + * Set up window for Node.js + */ + + var root = (typeof window !== 'undefined' ? window : {}); + + /* + * Parsing HTML strings + */ + + function canParseHTMLNatively () { + var Parser = root.DOMParser; + var canParse = false; + + // Adapted from https://gist.github.com/1129031 + // Firefox/Opera/IE throw errors on unsupported types + try { + // WebKit returns null on unsupported types + if (new Parser().parseFromString('', 'text/html')) { + canParse = true; + } + } catch (e) {} + + return canParse + } + + function createHTMLParser () { + var Parser = function () {}; + + { + if (shouldUseActiveX()) { + Parser.prototype.parseFromString = function (string) { + var doc = new window.ActiveXObject('htmlfile'); + doc.designMode = 'on'; // disable on-page scripts + doc.open(); + doc.write(string); + doc.close(); + return doc + }; + } else { + Parser.prototype.parseFromString = function (string) { + var doc = document.implementation.createHTMLDocument(''); + doc.open(); + doc.write(string); + doc.close(); + return doc + }; + } + } + return Parser + } + + function shouldUseActiveX () { + var useActiveX = false; + try { + document.implementation.createHTMLDocument('').open(); + } catch (e) { + if (window.ActiveXObject) useActiveX = true; + } + return useActiveX + } + + var HTMLParser = canParseHTMLNatively() ? root.DOMParser : createHTMLParser(); + + function RootNode (input) { + var root; + if (typeof input === 'string') { + var doc = htmlParser().parseFromString( + // DOM parsers arrange elements in the and . + // Wrapping in a custom element ensures elements are reliably arranged in + // a single element. + '' + input + '', + 'text/html' + ); + root = doc.getElementById('turndown-root'); + } else { + root = input.cloneNode(true); + } + collapseWhitespace({ + element: root, + isBlock: isBlock, + isVoid: isVoid + }); + + return root + } + + var _htmlParser; + function htmlParser () { + _htmlParser = _htmlParser || new HTMLParser(); + return _htmlParser + } + + function Node (node) { + node.isBlock = isBlock(node); + node.isCode = node.nodeName.toLowerCase() === 'code' || node.parentNode.isCode; + node.isBlank = isBlank(node); + node.flankingWhitespace = flankingWhitespace(node); + return node + } + + function isBlank (node) { + return ( + !isVoid(node) && + !isMeaningfulWhenBlank(node) && + /^\s*$/i.test(node.textContent) && + !hasVoid(node) && + !hasMeaningfulWhenBlank(node) + ) + } + + function flankingWhitespace (node) { + var leading = ''; + var trailing = ''; + + if (!node.isBlock) { + var hasLeading = /^\s/.test(node.textContent); + var hasTrailing = /\s$/.test(node.textContent); + var blankWithSpaces = node.isBlank && hasLeading && hasTrailing; + + if (hasLeading && !isFlankedByWhitespace('left', node)) { + leading = ' '; + } + + if (!blankWithSpaces && hasTrailing && !isFlankedByWhitespace('right', node)) { + trailing = ' '; + } + } + + return { leading: leading, trailing: trailing } + } + + function isFlankedByWhitespace (side, node) { + var sibling; + var regExp; + var isFlanked; + + if (side === 'left') { + sibling = node.previousSibling; + regExp = / $/; + } else { + sibling = node.nextSibling; + regExp = /^ /; + } + + if (sibling) { + if (sibling.nodeType === 3) { + isFlanked = regExp.test(sibling.nodeValue); + } else if (sibling.nodeType === 1 && !isBlock(sibling)) { + isFlanked = regExp.test(sibling.textContent); + } + } + return isFlanked + } + + var reduce = Array.prototype.reduce; + var leadingNewLinesRegExp = /^\n*/; + var trailingNewLinesRegExp = /\n*$/; + var escapes = [ + [/\\/g, '\\\\'], + [/\*/g, '\\*'], + [/^-/g, '\\-'], + [/^\+ /g, '\\+ '], + [/^(=+)/g, '\\$1'], + [/^(#{1,6}) /g, '\\$1 '], + [/`/g, '\\`'], + [/^~~~/g, '\\~~~'], + [/\[/g, '\\['], + [/\]/g, '\\]'], + [/^>/g, '\\>'], + [/_/g, '\\_'], + [/^(\d+)\. /g, '$1\\. '] + ]; + + function TurndownService (options) { + if (!(this instanceof TurndownService)) return new TurndownService(options) + + var defaults = { + rules: rules, + headingStyle: 'setext', + hr: '* * *', + bulletListMarker: '*', + codeBlockStyle: 'indented', + fence: '```', + emDelimiter: '_', + strongDelimiter: '**', + linkStyle: 'inlined', + linkReferenceStyle: 'full', + br: ' ', + blankReplacement: function (content, node) { + return node.isBlock ? '\n\n' : '' + }, + keepReplacement: function (content, node) { + return node.isBlock ? '\n\n' + node.outerHTML + '\n\n' : node.outerHTML + }, + defaultReplacement: function (content, node) { + return node.isBlock ? '\n\n' + content + '\n\n' : content + } + }; + this.options = extend({}, defaults, options); + this.rules = new Rules(this.options); + } + + TurndownService.prototype = { + /** + * The entry point for converting a string or DOM node to Markdown + * @public + * @param {String|HTMLElement} input The string or DOM node to convert + * @returns A Markdown representation of the input + * @type String + */ + + turndown: function (input) { + if (!canConvert(input)) { + throw new TypeError( + input + ' is not a string, or an element/document/fragment node.' + ) + } + + if (input === '') return '' + + var output = process.call(this, new RootNode(input)); + return postProcess.call(this, output) + }, + + /** + * Add one or more plugins + * @public + * @param {Function|Array} plugin The plugin or array of plugins to add + * @returns The Turndown instance for chaining + * @type Object + */ + + use: function (plugin) { + if (Array.isArray(plugin)) { + for (var i = 0; i < plugin.length; i++) this.use(plugin[i]); + } else if (typeof plugin === 'function') { + plugin(this); + } else { + throw new TypeError('plugin must be a Function or an Array of Functions') + } + return this + }, + + /** + * Adds a rule + * @public + * @param {String} key The unique key of the rule + * @param {Object} rule The rule + * @returns The Turndown instance for chaining + * @type Object + */ + + addRule: function (key, rule) { + this.rules.add(key, rule); + return this + }, + + /** + * Keep a node (as HTML) that matches the filter + * @public + * @param {String|Array|Function} filter The unique key of the rule + * @returns The Turndown instance for chaining + * @type Object + */ + + keep: function (filter) { + this.rules.keep(filter); + return this + }, + + /** + * Remove a node that matches the filter + * @public + * @param {String|Array|Function} filter The unique key of the rule + * @returns The Turndown instance for chaining + * @type Object + */ + + remove: function (filter) { + this.rules.remove(filter); + return this + }, + + /** + * Escapes Markdown syntax + * @public + * @param {String} string The string to escape + * @returns A string with Markdown syntax escaped + * @type String + */ + + escape: function (string) { + return escapes.reduce(function (accumulator, escape) { + return accumulator.replace(escape[0], escape[1]) + }, string) + } + }; + + /** + * Reduces a DOM node down to its Markdown string equivalent + * @private + * @param {HTMLElement} parentNode The node to convert + * @returns A Markdown representation of the node + * @type String + */ + + function process (parentNode) { + var self = this; + return reduce.call(parentNode.childNodes, function (output, node) { + node = new Node(node); + + var replacement = ''; + if (node.nodeType === 3) { + replacement = node.isCode ? node.nodeValue : self.escape(node.nodeValue); + } else if (node.nodeType === 1) { + replacement = replacementForNode.call(self, node); + } + + return join(output, replacement) + }, '') + } + + /** + * Appends strings as each rule requires and trims the output + * @private + * @param {String} output The conversion output + * @returns A trimmed version of the ouput + * @type String + */ + + function postProcess (output) { + var self = this; + this.rules.forEach(function (rule) { + if (typeof rule.append === 'function') { + output = join(output, rule.append(self.options)); + } + }); + + return output.replace(/^[\t\r\n]+/, '').replace(/[\t\r\n\s]+$/, '') + } + + /** + * Converts an element node to its Markdown equivalent + * @private + * @param {HTMLElement} node The node to convert + * @returns A Markdown representation of the node + * @type String + */ + + function replacementForNode (node) { + var rule = this.rules.forNode(node); + var content = process.call(this, node); + var whitespace = node.flankingWhitespace; + if (whitespace.leading || whitespace.trailing) content = content.trim(); + return ( + whitespace.leading + + rule.replacement(content, node, this.options) + + whitespace.trailing + ) + } + + /** + * Determines the new lines between the current output and the replacement + * @private + * @param {String} output The current conversion output + * @param {String} replacement The string to append to the output + * @returns The whitespace to separate the current output and the replacement + * @type String + */ + + function separatingNewlines (output, replacement) { + var newlines = [ + output.match(trailingNewLinesRegExp)[0], + replacement.match(leadingNewLinesRegExp)[0] + ].sort(); + var maxNewlines = newlines[newlines.length - 1]; + return maxNewlines.length < 2 ? maxNewlines : '\n\n' + } + + function join (string1, string2) { + var separator = separatingNewlines(string1, string2); + + // Remove trailing/leading newlines and replace with separator + string1 = string1.replace(trailingNewLinesRegExp, ''); + string2 = string2.replace(leadingNewLinesRegExp, ''); + + return string1 + separator + string2 + } + + /** + * Determines whether an input can be converted + * @private + * @param {String|HTMLElement} input Describe this parameter + * @returns Describe what it returns + * @type String|Object|Array|Boolean|Number + */ + + function canConvert (input) { + return ( + input != null && ( + typeof input === 'string' || + (input.nodeType && ( + input.nodeType === 1 || input.nodeType === 9 || input.nodeType === 11 + )) + ) + ) + } + + return TurndownService; + +}))); diff --git a/www/pad/export.js b/www/pad/export.js index f1e3497c9..db9f5ac37 100644 --- a/www/pad/export.js +++ b/www/pad/export.js @@ -1,12 +1,24 @@ define([ 'jquery', '/common/common-util.js', + '/common/diffMarked.js', + '/common/hyperscript.js', '/bower_components/hyperjson/hyperjson.js', '/bower_components/nthen/index.js', -], function ($, Util, Hyperjson, nThen) { + '/lib/turndown.browser.umd.js' +], function ($, Util, DiffMd, h, Hyperjson, nThen, Turndown) { var module = { ext: '.html', // default - exts: ['.html', '.doc'] + exts: ['.html', '.md', '.doc'] + }; + + module.importMd = function (md, common) { + var html = DiffMd.render(md, true, false, true); + var div = h('div#cp-temp'); + DiffMd.apply(html, $(div), common); + var body = h('body'); + body.innerHTML = div.innerHTML; + return body; }; var exportMediaTags = function (inner, cb) { @@ -77,6 +89,15 @@ define([ }); return void cb(blob); } + if (ext === ".md") { + var md = Turndown({ + headingStyle: 'atx' + }).turndown(toExport); + var blob = new Blob([md], { + type: 'text/markdown;charset=utf-8' + }); + return void cb(blob); + } var html = module.getHTML(toExport); cb(new Blob([ html ], { type: "text/html;charset=utf-8" })); }); diff --git a/www/pad/inner.js b/www/pad/inner.js index 47ab5dbe3..47bd7f7f7 100644 --- a/www/pad/inner.js +++ b/www/pad/inner.js @@ -1169,7 +1169,14 @@ define([ }); cb($dom[0]); }; - framework.setFileImporter({ accept: 'text/html' }, function(content, f, cb) { + framework.setFileImporter({ accept: ['.md', 'text/html'] }, function(content, f, cb) { + if (!f) { return; } + if (/\.md$/.test(f.name)) { + var mdDom = Exporter.importMd(content, framework._.sfCommon); + return importMediaTags(mdDom, function(dom) { + cb(Hyperjson.fromDOM(dom)); + }); + } importMediaTags(domFromHTML(content).body, function(dom) { cb(Hyperjson.fromDOM(dom)); }); From 544e5bcbfe580a6a22289f3821a9c5b792b40acc Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 22 Jun 2021 12:08:03 +0200 Subject: [PATCH 06/79] Add converter app --- www/common/application_config_internal.js | 2 +- www/convert/app-convert.less | 16 ++ www/convert/file-crypto.js | 209 +++++++++++++++++ www/convert/index.html | 12 + www/convert/inner.html | 18 ++ www/convert/inner.js | 259 ++++++++++++++++++++++ www/convert/main.js | 28 +++ 7 files changed, 543 insertions(+), 1 deletion(-) create mode 100644 www/convert/app-convert.less create mode 100644 www/convert/file-crypto.js create mode 100644 www/convert/index.html create mode 100644 www/convert/inner.html create mode 100644 www/convert/inner.js create mode 100644 www/convert/main.js diff --git a/www/common/application_config_internal.js b/www/common/application_config_internal.js index ad5f726f8..68ba4ed06 100644 --- a/www/common/application_config_internal.js +++ b/www/common/application_config_internal.js @@ -12,7 +12,7 @@ define(function() { * You should never remove the drive from this list. */ AppConfig.availablePadTypes = ['drive', 'teams', 'pad', 'sheet', 'code', 'slide', 'poll', 'kanban', 'whiteboard', - /*'doc', 'presentation',*/ 'file', /*'todo',*/ 'contacts', 'form']; + /*'doc', 'presentation',*/ 'file', /*'todo',*/ 'contacts', 'form', 'convert']; /* The registered only types are apps restricted to registered users. * You should never remove apps from this list unless you know what you're doing. The apps * listed here by default can't work without a user account. diff --git a/www/convert/app-convert.less b/www/convert/app-convert.less new file mode 100644 index 000000000..c5813641a --- /dev/null +++ b/www/convert/app-convert.less @@ -0,0 +1,16 @@ +@import (reference) '../../customize/src/less2/include/framework.less'; +@import (reference) '../../customize/src/less2/include/sidebar-layout.less'; + +&.cp-app-convert { + + .framework_min_main( + @bg-color: @colortheme_apps[default], + ); + .sidebar-layout_main(); + + // body + display: flex; + flex-flow: column; + background-color: @cp_app-bg; + +} diff --git a/www/convert/file-crypto.js b/www/convert/file-crypto.js new file mode 100644 index 000000000..6a0c08816 --- /dev/null +++ b/www/convert/file-crypto.js @@ -0,0 +1,209 @@ +define([ + '/bower_components/tweetnacl/nacl-fast.min.js', +], function () { + var Nacl = window.nacl; + //var PARANOIA = true; + + var plainChunkLength = 128 * 1024; + var cypherChunkLength = 131088; + + var computeEncryptedSize = function (bytes, meta) { + var metasize = Nacl.util.decodeUTF8(JSON.stringify(meta)).length; + var chunks = Math.ceil(bytes / plainChunkLength); + return metasize + 18 + (chunks * 16) + bytes; + }; + + var encodePrefix = function (p) { + return [ + 65280, // 255 << 8 + 255, + ].map(function (n, i) { + return (p & n) >> ((1 - i) * 8); + }); + }; + var decodePrefix = function (A) { + return (A[0] << 8) | A[1]; + }; + + var slice = function (A) { + return Array.prototype.slice.call(A); + }; + + var createNonce = function () { + return new Uint8Array(new Array(24).fill(0)); + }; + + var increment = function (N) { + var l = N.length; + while (l-- > 1) { + /* our linter suspects this is unsafe because we lack types + but as long as this is only used on nonces, it should be safe */ + if (N[l] !== 255) { return void N[l]++; } // jshint ignore:line + if (l === 0) { throw new Error('E_NONCE_TOO_LARGE'); } + N[l] = 0; + } + }; + + var joinChunks = function (chunks) { + return new Blob(chunks); + }; + + var decrypt = function (u8, key, done, progress) { + var MAX = u8.length; + var _progress = function (offset) { + if (typeof(progress) !== 'function') { return; } + progress(Math.min(1, offset / MAX)); + }; + + var nonce = createNonce(); + var i = 0; + + var prefix = u8.subarray(0, 2); + var metadataLength = decodePrefix(prefix); + + var res = { + metadata: undefined, + }; + + var cancelled = false; + var cancel = function () { + cancelled = true; + }; + + var metaBox = new Uint8Array(u8.subarray(2, 2 + metadataLength)); + + var metaChunk = Nacl.secretbox.open(metaBox, nonce, key); + increment(nonce); + + try { + res.metadata = JSON.parse(Nacl.util.encodeUTF8(metaChunk)); + } catch (e) { + return window.setTimeout(function () { + done('E_METADATA_DECRYPTION'); + }); + } + + if (!res.metadata) { + return void setTimeout(function () { + done('NO_METADATA'); + }); + } + + var takeChunk = function (cb) { + setTimeout(function () { + var start = i * cypherChunkLength + 2 + metadataLength; + var end = start + cypherChunkLength; + i++; + var box = new Uint8Array(u8.subarray(start, end)); + + // decrypt the chunk + var plaintext = Nacl.secretbox.open(box, nonce, key); + increment(nonce); + + if (!plaintext) { return cb('DECRYPTION_ERROR'); } + + _progress(end); + cb(void 0, plaintext); + }); + }; + + var chunks = []; + + var again = function () { + if (cancelled) { return; } + takeChunk(function (e, plaintext) { + if (e) { + return setTimeout(function () { + done(e); + }); + } + if (plaintext) { + if ((2 + metadataLength + i * cypherChunkLength) < u8.length) { // not done + chunks.push(plaintext); + return setTimeout(again); + } + chunks.push(plaintext); + res.content = joinChunks(chunks); + return done(void 0, res); + } + done('UNEXPECTED_ENDING'); + }); + }; + + again(); + + return { + cancel: cancel + }; + }; + + // metadata + /* { filename: 'raccoon.jpg', type: 'image/jpeg' } */ + var encrypt = function (u8, metadata, key) { + var nonce = createNonce(); + + // encode metadata + var plaintext = Nacl.util.decodeUTF8(JSON.stringify(metadata)); + + // if metadata is too large, drop the thumbnail. + if (plaintext.length > 65535) { + var temp = JSON.parse(JSON.stringify(metadata)); + delete metadata.thumbnail; + plaintext = Nacl.util.decodeUTF8(JSON.stringify(temp)); + } + + var i = 0; + + var state = 0; + var next = function (cb) { + if (state === 2) { return void setTimeout(cb); } + + var start; + var end; + var part; + var box; + + if (state === 0) { // metadata... + part = new Uint8Array(plaintext); + box = Nacl.secretbox(part, nonce, key); + increment(nonce); + + if (box.length > 65535) { + return void cb('METADATA_TOO_LARGE'); + } + var prefixed = new Uint8Array(encodePrefix(box.length) + .concat(slice(box))); + state++; + + return void setTimeout(function () { + cb(void 0, prefixed); + }); + } + + // encrypt the rest of the file... + start = i * plainChunkLength; + end = start + plainChunkLength; + + part = u8.subarray(start, end); + box = Nacl.secretbox(part, nonce, key); + increment(nonce); + i++; + + // regular data is done + if (i * plainChunkLength >= u8.length) { state = 2; } + + setTimeout(function () { + cb(void 0, box); + }); + }; + + return next; + }; + + return { + decrypt: decrypt, + encrypt: encrypt, + joinChunks: joinChunks, + computeEncryptedSize: computeEncryptedSize, + }; +}); diff --git a/www/convert/index.html b/www/convert/index.html new file mode 100644 index 000000000..96a3cce86 --- /dev/null +++ b/www/convert/index.html @@ -0,0 +1,12 @@ + + + + CryptPad + + + + + + + + diff --git a/www/convert/inner.html b/www/convert/inner.html new file mode 100644 index 000000000..206b85722 --- /dev/null +++ b/www/convert/inner.html @@ -0,0 +1,18 @@ + + + + + + + + +
+
+ + + diff --git a/www/convert/inner.js b/www/convert/inner.js new file mode 100644 index 000000000..d8eca9af2 --- /dev/null +++ b/www/convert/inner.js @@ -0,0 +1,259 @@ +define([ + 'jquery', + '/api/config', + '/bower_components/chainpad-crypto/crypto.js', + '/common/toolbar.js', + '/bower_components/nthen/index.js', + '/common/sframe-common.js', + '/common/hyperscript.js', + '/customize/messages.js', + '/common/common-interface.js', + '/common/common-util.js', + + '/bower_components/file-saver/FileSaver.min.js', + 'css!/bower_components/bootstrap/dist/css/bootstrap.min.css', + 'css!/bower_components/components-font-awesome/css/font-awesome.min.css', + 'less!/convert/app-convert.less', +], function ( + $, + ApiConfig, + Crypto, + Toolbar, + nThen, + SFCommon, + h, + Messages, + UI, + Util + ) +{ + var APP = {}; + + var common; + var sFrameChan; + + var debug = console.debug; + + var x2tReady = Util.mkEvent(true); + var x2tInitialized = false; + var x2tInit = function(x2t) { + debug("x2t mount"); + // x2t.FS.mount(x2t.MEMFS, {} , '/'); + x2t.FS.mkdir('/working'); + x2t.FS.mkdir('/working/media'); + x2t.FS.mkdir('/working/fonts'); + x2tInitialized = true; + x2tReady.fire(); + //fetchFonts(x2t); + debug("x2t mount done"); + }; + var getX2t = function (cb) { + // XXX require http headers on firefox... + require(['/common/onlyoffice/x2t/x2t.js'], function() { // FIXME why does this fail without an access-control-allow-origin header? + var x2t = window.Module; + x2t.run(); + if (x2tInitialized) { + debug("x2t runtime already initialized"); + return void x2tReady.reg(function () { + cb(x2t); + }); + } + + x2t.onRuntimeInitialized = function() { + debug("x2t in runtime initialized"); + // Init x2t js module + x2tInit(x2t); + x2tReady.reg(function () { + cb(x2t); + }); + }; + }); + }; + /* + Converting Data + + This function converts a data in a specific format to the outputformat + The filename extension needs to represent the input format + Example: fileName=cryptpad.bin outputFormat=xlsx + */ + var getFormatId = function (ext) { + // Sheets + if (ext === 'xlsx') { return 257; } + if (ext === 'xls') { return 258; } + if (ext === 'ods') { return 259; } + if (ext === 'csv') { return 260; } + if (ext === 'pdf') { return 513; } + // Docs + if (ext === 'docx') { return 65; } + if (ext === 'doc') { return 66; } + if (ext === 'odt') { return 67; } + if (ext === 'txt') { return 69; } + if (ext === 'html') { return 70; } + + // Slides + if (ext === 'pptx') { return 129; } + if (ext === 'ppt') { return 130; } + if (ext === 'odp') { return 131; } + + return; + }; + var getFromId = function (ext) { + var id = getFormatId(ext); + if (!id) { return ''; } + return ''+id+''; + }; + var getToId = function (ext) { + var id = getFormatId(ext); + if (!id) { return ''; } + return ''+id+''; + }; + var x2tConvertDataInternal = function(x2t, data, fileName, outputFormat) { + debug("Converting Data for " + fileName + " to " + outputFormat); + + var inputFormat = fileName.split('.').pop(); + + x2t.FS.writeFile('/working/' + fileName, data); + var params = "" + + "" + + "/working/" + fileName + "" + + "/working/" + fileName + "." + outputFormat + "" + + getFromId(inputFormat) + + getToId(outputFormat) + + "false" + + ""; + // writing params file to mounted working disk (in memory) + x2t.FS.writeFile('/working/params.xml', params); + // running conversion + x2t.ccall("runX2T", ["number"], ["string"], ["/working/params.xml"]); + // reading output file from working disk (in memory) + var result; + try { + result = x2t.FS.readFile('/working/' + fileName + "." + outputFormat); + } catch (e) { + console.error(e, x2t.FS); + debug("Failed reading converted file"); + UI.warn(Messages.error); + return ""; + } + return result; + }; + var x2tConverter = function (typeSrc, typeTarget) { + return function (data, name, cb) { + getX2t(function (x2t) { + if (typeSrc === 'ods') { + data = x2tConvertDataInternal(x2t, data, name, 'xlsx'); + name += '.xlsx'; + } + if (typeSrc === 'odt') { + data = x2tConvertDataInternal(x2t, data, name, 'docx'); + name += '.docx'; + } + if (typeSrc === 'odp') { + data = x2tConvertDataInternal(x2t, data, name, 'pptx'); + name += '.pptx'; + } + cb(x2tConvertDataInternal(x2t, data, name, typeTarget)); + }); + }; + }; + + var CONVERTERS = { + xlsx: { + //pdf: x2tConverter('xlsx', 'pdf'), + ods: x2tConverter('xlsx', 'ods'), + bin: x2tConverter('xlsx', 'bin'), + }, + ods: { + //pdf: x2tConverter('ods', 'pdf'), + xlsx: x2tConverter('ods', 'xlsx'), + bin: x2tConverter('ods', 'bin'), + }, + odt: { + docx: x2tConverter('odt', 'docx'), + txt: x2tConverter('odt', 'txt'), + bin: x2tConverter('odt', 'bin'), + }, + docx: { + odt: x2tConverter('docx', 'odt'), + txt: x2tConverter('docx', 'txt'), + bin: x2tConverter('docx', 'bin'), + }, + txt: { + odt: x2tConverter('txt', 'odt'), + docx: x2tConverter('txt', 'docx'), + bin: x2tConverter('txt', 'bin'), + }, + odp: { + pptx: x2tConverter('odp', 'pptx'), + bin: x2tConverter('odp', 'bin'), + }, + pptx: { + odp: x2tConverter('pptx', 'odp'), + bin: x2tConverter('pptx', 'bin'), + }, + }; + + Messages.convertPage = "Convert"; // XXX + Messages.convert_hint = "Pick the file you want to convert. The list of output format will be visible afterward."; + + var createToolbar = function () { + var displayed = ['useradmin', 'newpad', 'limit', 'pageTitle', 'notifications']; + var configTb = { + displayed: displayed, + sfCommon: common, + $container: APP.$toolbar, + pageTitle: Messages.convertPage, + metadataMgr: common.getMetadataMgr(), + }; + APP.toolbar = Toolbar.create(configTb); + APP.toolbar.$rightside.hide(); + }; + + nThen(function (waitFor) { + $(waitFor(UI.addLoadingScreen)); + SFCommon.create(waitFor(function (c) { APP.common = common = c; })); + }).nThen(function (waitFor) { + APP.$container = $('#cp-sidebarlayout-container'); + APP.$toolbar = $('#cp-toolbar'); + APP.$leftside = $('
', {id: 'cp-sidebarlayout-leftside'}).appendTo(APP.$container); + APP.$rightside = $('
', {id: 'cp-sidebarlayout-rightside'}).appendTo(APP.$container); + sFrameChan = common.getSframeChannel(); + sFrameChan.onReady(waitFor()); + }).nThen(function (/*waitFor*/) { + createToolbar(); + + var hint = h('p.cp-convert-hint', Messages.convert_hint); + + var picker = h('input', { + type: 'file' + }); + APP.$rightside.append([hint, picker]); + + $(picker).on('change', function () { + var file = picker.files[0]; + var name = file && file.name; + var reader = new FileReader(); + var parsed = file && file.name && /.+\.([^.]+)$/.exec(file.name); + var ext = parsed && parsed[1]; + reader.onload = function (e) { + if (CONVERTERS[ext]) { + Object.keys(CONVERTERS[ext]).forEach(function (to) { + var button = h('button.btn', to); + $(button).click(function () { + CONVERTERS[ext][to](new Uint8Array(e.target.result), name, function (a) { + var n = name.slice(0, -ext.length) + to; + var blob = new Blob([a], {type: "application/bin;charset=utf-8"}); + window.saveAs(blob, n); + }); + + }).appendTo(APP.$rightside); + }); + } + }; + reader.readAsArrayBuffer(file, 'application/octet-stream'); + }); + + UI.removeLoadingScreen(); + + }); +}); diff --git a/www/convert/main.js b/www/convert/main.js new file mode 100644 index 000000000..236290b13 --- /dev/null +++ b/www/convert/main.js @@ -0,0 +1,28 @@ +// Load #1, load as little as possible because we are in a race to get the loading screen up. +define([ + '/bower_components/nthen/index.js', + '/api/config', + '/common/dom-ready.js', + '/common/sframe-common-outer.js' +], function (nThen, ApiConfig, DomReady, SFCommonO) { + + // Loaded in load #2 + nThen(function (waitFor) { + DomReady.onReady(waitFor()); + }).nThen(function (waitFor) { + SFCommonO.initIframe(waitFor, true); + }).nThen(function (/*waitFor*/) { + var category; + if (window.location.hash) { + category = window.location.hash.slice(1); + window.location.hash = ''; + } + var addData = function (obj) { + if (category) { obj.category = category; } + }; + SFCommonO.start({ + noRealtime: true, + addData: addData + }); + }); +}); From 04fc838ef37537cc77b00282986f7b5b505e1813 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 22 Jun 2021 12:08:10 +0200 Subject: [PATCH 07/79] lint compliance --- www/pad/export.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/www/pad/export.js b/www/pad/export.js index db9f5ac37..01ac27d7e 100644 --- a/www/pad/export.js +++ b/www/pad/export.js @@ -93,10 +93,10 @@ define([ var md = Turndown({ headingStyle: 'atx' }).turndown(toExport); - var blob = new Blob([md], { + var mdBlob = new Blob([md], { type: 'text/markdown;charset=utf-8' }); - return void cb(blob); + return void cb(mdBlob); } var html = module.getHTML(toExport); cb(new Blob([ html ], { type: "text/html;charset=utf-8" })); From 5d156dd346d6b5ce644a81953f7bc1ebcb4c4b80 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 22 Jun 2021 12:24:41 +0200 Subject: [PATCH 08/79] Fix base64 image detection in pads --- www/pad/inner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/pad/inner.js b/www/pad/inner.js index 47bd7f7f7..ae22af23c 100644 --- a/www/pad/inner.js +++ b/www/pad/inner.js @@ -1115,7 +1115,7 @@ define([ framework._.sfCommon.isPadStored(function(err, val) { if (!val) { return; } - var b64images = $inner.find('img[src^="data:image"]:not(.cke_reset)'); + var b64images = $inner.find('img[src^="data:image"]:not(.cke_reset), img[src^="data:application/octet-stream"]:not(.cke_reset)'); if (b64images.length && framework._.sfCommon.isLoggedIn()) { var no = h('button.cp-corner-cancel', Messages.cancel); var yes = h('button.cp-corner-primary', Messages.ok); From f5e91ef3ef4c34c97178825c06fec2e81808d741 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 22 Jun 2021 16:16:32 +0530 Subject: [PATCH 09/79] provide installMethod detail in server telemetry --- config/config.example.js | 9 +++++++++ lib/env.js | 1 + lib/stats.js | 1 + 3 files changed, 11 insertions(+) diff --git a/config/config.example.js b/config/config.example.js index 69f0b1e91..96914fa92 100644 --- a/config/config.example.js +++ b/config/config.example.js @@ -276,4 +276,13 @@ module.exports = { * (false by default) */ verbose: false, + + /* Surplus information: + * + * 'installMethod' is included in server telemetry to voluntarily + * indicate how many instances are using unofficial installation methods + * such as Docker. + * + */ + installMethod: 'unspecified', }; diff --git a/lib/env.js b/lib/env.js index 6f1717c09..6b033fa16 100644 --- a/lib/env.js +++ b/lib/env.js @@ -20,6 +20,7 @@ var canonicalizeOrigin = function (s) { module.exports.create = function (config) { const Env = { version: Package.version, + installMethod: config.installMethod || undefined, httpUnsafeOrigin: canonicalizeOrigin(config.httpUnsafeOrigin), httpSafeOrigin: canonicalizeOrigin(config.httpSafeOrigin), diff --git a/lib/stats.js b/lib/stats.js index d1da0e202..da820f7b8 100644 --- a/lib/stats.js +++ b/lib/stats.js @@ -4,6 +4,7 @@ const Stats = module.exports; Stats.instanceData = function (Env) { var data = { version: Env.version, + installMethod: Env.installMethod, domain: Env.myDomain, subdomain: Env.mySubdomain, From 433470cf401cf0f54d24c91a9adc85eb5d21fc2b Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 23 Jun 2021 07:54:28 +0530 Subject: [PATCH 10/79] check that server responses don't contain 'Server' headers if they do, check that the server is NGINX. --- www/checkup/main.js | 48 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/www/checkup/main.js b/www/checkup/main.js index 925cd4b7b..2639ff156 100644 --- a/www/checkup/main.js +++ b/www/checkup/main.js @@ -732,6 +732,54 @@ define([ cb(isHTTPS(trimmedUnsafe) && isHTTPS(trimmedSafe)); }); + assert(function (cb, msg) { + setWarningClass(msg); + $.ajax(cacheBuster('/'), { + dataType: 'text', + complete: function (xhr) { + var serverToken = xhr.getResponseHeader('server'); + if (serverToken === null) { return void cb(true); } + + var lowered = (serverToken || '').toLowerCase(); + var family; + + ['Apache', 'Caddy', 'NGINX'].some(function (pattern) { + if (lowered.indexOf(pattern.toLowerCase()) !== -1) { + family = pattern; + return true; + } + }); + + var text = [ + "This instance is set to respond with an HTTP ", + code("server"), + " header. This information can make it easier for attackers to find and exploit known vulnerabilities. ", + ]; + + + if (family === 'NGINX') { + msg.appendChild(h('span', text.concat([ + "This can be addressed by setting ", + code("server_tokens off"), + " in your global NGINX config." + ]))); + return void cb(serverToken); + } + + // handle other + msg.appendChild(h('span', text.concat([ + "In this case, it appears that the host server is running ", + code(serverToken), + " instead of ", + code("NGINX"), + " as recommended. As such, you may not benefit from the latest security enhancements that are tested and maintained by the CryptPad development team.", + ]))); + + cb(serverToken); + } + }); + }); + if (false) { assert(function (cb, msg) { msg.innerText = 'fake test to simulate failure'; From 6ddcbb948ef7e7f8eb9ad35f936e00ce4f2df3d0 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 23 Jun 2021 09:32:58 +0530 Subject: [PATCH 11/79] guard against markdown images with double-quotes in their href --- www/common/diffMarked.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index cb95f37aa..6be2f0e7c 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -267,7 +267,7 @@ define([ }; renderer.image = function (href, title, text) { - if (href.slice(0,6) === '/file/') { + if (href.slice(0,6) === '/file/') { // XXX this has been deprecated for about 3 years... use the same inline image handler as below? // DEPRECATED // Mediatag using markdown syntax should not be used anymore so they don't support // password-protected files @@ -283,12 +283,14 @@ define([ mt += ''; return mt; } - var out = '' + text + '' : '>'; - return out; + + var img = h('img.cp-inline-img', { + src: href || '', + title: title || '', + alt: text || '', + }); + + return img.outerHTML; }; restrictedRenderer.image = renderer.image; From 94abd631a25af7f252379df91444d7434b48059e Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 23 Jun 2021 09:33:39 +0530 Subject: [PATCH 12/79] prototype suppression of markdown images --- .../src/less2/include/markdown.less | 9 ++++++ www/common/diffMarked.js | 29 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/customize.dist/src/less2/include/markdown.less b/customize.dist/src/less2/include/markdown.less index 3943f128d..43d9debaf 100644 --- a/customize.dist/src/less2/include/markdown.less +++ b/customize.dist/src/less2/include/markdown.less @@ -154,6 +154,15 @@ color: @cp_markdown-block-fg; text-align: left; } + + span.cp-inline-img-warning { + display: inline-block; + border: 1px solid red; + a, br, strong { + border: none; + } + } + //.cp-inline-img { } } .markdown_cryptpad() { diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index 6be2f0e7c..770f4daad 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -723,6 +723,35 @@ define([ if (target) { target.scrollIntoView(); } }); + // replace remote images with links to those images + $content.find('img.cp-inline-img').each(function (index, el) { + var link = h('a', { + href: el.src, //common.getBounceURL(el.src), // XXX + target: '_blank', + rel: 'noopener noreferrer', + title: el.src, + }, [ + 'open image at ', + h('strong', el.src), + ]); + + link.onclick = function (ev) { + ev.preventDefault(); + ev.stopPropagation(); + common.openURL(el.src); + }; + + var warning = h('span.cp-inline-img-warning', [ + "CryptPad disallows unencrypted images", + h('br'), + h('br'), + link, + ]); + + var parent = el.parentElement; + parent.replaceChild(warning, el); + }); + // loop over plugin elements in the rendered content $content.find('pre[data-plugin]').each(function (index, el) { var plugin = plugins[el.getAttribute('data-plugin')]; From 761e27cac7582502ab2cb36465beb9b0f5d999f5 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 23 Jun 2021 16:19:48 +0530 Subject: [PATCH 13/79] WIP: transform inline css into less and scope it to the preview pane --- www/common/diffMarked.js | 75 +++++++++++++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 8 deletions(-) diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index cb95f37aa..9312924ea 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -8,11 +8,12 @@ define([ '/common/inner/common-mediatag.js', '/common/media-tag.js', '/customize/messages.js', + '/common/less.min.js', '/common/highlight/highlight.pack.js', '/lib/diff-dom/diffDOM.js', '/bower_components/tweetnacl/nacl-fast.min.js', 'css!/common/highlight/styles/'+ (window.CryptPad_theme === 'dark' ? 'dark.css' : 'github.css') -],function ($, ApiConfig, Marked, Hash, Util, h, MT, MediaTag, Messages) { +],function ($, ApiConfig, Marked, Hash, Util, h, MT, MediaTag, Messages, Less) { var DiffMd = {}; var Highlight = window.hljs; @@ -473,6 +474,43 @@ define([ } }; + var rendered_less = {}; // XXX this never gets evicted, so it's a memory leak + + var applyCSS = function (el, css) { + var style = h('style'); + style.appendChild(document.createTextNode(css)); + el.innerText = ''; + el.appendChild(style); + }; + + var canonicalizeLess = function (source) { + return source.replace(/\/\/[^\n]*/g, '').replace(/^[ \t]*$/g, '').replace(/\n+/g, ''); + }; + + plugins.less = { // XXX + name: 'less', + attr: 'less-src', + render: function renderLess ($el, opt) { + var src = canonicalizeLess(($el.text() || '').trim()); + + console.log(src); + if (!src) { return; } + var el = $el[0]; + if (rendered_less[src]) { // XXX janky cache instead of using the same methodology as other plugins... + return void applyCSS(el, rendered_less[src]); + } + + var scope = opt.scope.attr('id') || 'cp-app-code-preview-content'; + var scoped_src = '#' + scope + ' { ' + src + '}'; + console.error("RENDERING LESS", opt.cache); + Less.render(scoped_src, {}, function (err, result) { + if (err) { return void console.error(err); } + var css = rendered_less[src] = result.css; + applyCSS(el, css); + }); + }, + }; + var getAvailableCachedElement = function ($content, cache, src) { var cached = cache[src]; if (!Array.isArray(cached)) { return; } @@ -485,7 +523,8 @@ define([ } }; - var cacheRenderedElement = function (cache, src, el) { + var cacheRenderedElement = function (cache, src, el) { // XXX + console.log("CACHING", cache, src, el); if (Array.isArray(cache[src])) { cache[src].push(el); } else { @@ -546,7 +585,9 @@ define([ // caching their source as you go $(newDomFixed).find('pre[data-plugin]').each(function (index, el) { if (el.childNodes.length === 1 && el.childNodes[0].nodeType === 3) { - var plugin = plugins[el.getAttribute('data-plugin')]; + var type = el.getAttribute('data-plugin'); + var plugin = plugins[type]; + console.log(type); if (!plugin) { return; } var src = canonicalizeMermaidSource(el.childNodes[0].wholeText); el.setAttribute(plugin.attr, src); @@ -559,7 +600,8 @@ define([ var scrollTop = $parent.scrollTop(); // iterate over rendered mermaid charts $content.find('pre[data-plugin]:not([processed="true"])').each(function (index, el) { - var plugin = plugins[el.getAttribute('data-plugin')]; + var type = el.getAttribute('data-plugin'); + var plugin = plugins[type]; if (!plugin) { return; } // retrieve the attached source code which it was drawn @@ -721,9 +763,21 @@ define([ if (target) { target.scrollIntoView(); } }); + $content.find('style').each(function (index, el) { // XXX + var parent = el.parentElement; + var pre = h('pre', { + 'data-plugin': 'less', + 'less-src': el.innerText, + style: 'display: none', + }, el.innerText); + parent.replaceChild(pre, el); + }); + // loop over plugin elements in the rendered content - $content.find('pre[data-plugin]').each(function (index, el) { - var plugin = plugins[el.getAttribute('data-plugin')]; + $content.find('pre[data-plugin]').each(function (index, el) { // XXX + var type = el.getAttribute('data-plugin'); + var plugin = plugins[type]; + if (!plugin) { return; } var $el = $(el); $el.off('contextmenu').on('contextmenu', function (e) { @@ -742,13 +796,18 @@ define([ // you can assume that the index of your rendered charts matches that // of those in the markdown source. var src = plugin.source[index]; - el.setAttribute(plugin.attr, src); + if (src) { + el.setAttribute(plugin.attr, src); + } var cached = getAvailableCachedElement($content, plugin.cache, src); // check if you had cached a pre-rendered instance of the supplied source if (typeof(cached) !== 'object') { try { - plugin.render($el); + plugin.render($el, { + scope: $content, // XXX + cache: plugin.cache, + }); } catch (e) { console.error(e); } return; } From c4aeddeee5ac1cfbbf907cf6a428b38fadff2e2e Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 24 Jun 2021 11:48:55 +0530 Subject: [PATCH 14/79] bugfixes for the less cache --- www/common/diffMarked.js | 72 +++++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index 9312924ea..d2877adbd 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -474,8 +474,6 @@ define([ } }; - var rendered_less = {}; // XXX this never gets evicted, so it's a memory leak - var applyCSS = function (el, css) { var style = h('style'); style.appendChild(document.createTextNode(css)); @@ -483,28 +481,62 @@ define([ el.appendChild(style); }; + // trim non-functional text from less input so that + // the compiler is only triggered when there has been a functional change var canonicalizeLess = function (source) { - return source.replace(/\/\/[^\n]*/g, '').replace(/^[ \t]*$/g, '').replace(/\n+/g, ''); - }; + return (source || '') + // leading and trailing spaces are irrelevant + .trim() + // line comments are easy to disregard + .replace(/\/\/[^\n]*/g, '') + // lines with nothing but spaces and tabs can be ignored + .replace(/^[ \t]*$/g, '') + // consecutive newlines make no difference + .replace(/\n+/g, ''); + }; + + var rendered_less = {}; + var getRenderedLess = (function () { + var timeouts = {}; + return function (src) { + if (!rendered_less[src]) { return; } + if (timeouts[src]) { + clearTimeout(timeouts[src]); + } + // avoid memory leaks by deleting cached content + // 15s after it was last accessed + timeouts[src] = setTimeout(function () { + delete rendered_less[src]; + delete timeouts[src]; + }, 15000); + return rendered_less[src]; + }; + }()); - plugins.less = { // XXX + plugins.less = { name: 'less', attr: 'less-src', render: function renderLess ($el, opt) { - var src = canonicalizeLess(($el.text() || '').trim()); - - console.log(src); + var src = canonicalizeLess($el.text()); if (!src) { return; } var el = $el[0]; - if (rendered_less[src]) { // XXX janky cache instead of using the same methodology as other plugins... - return void applyCSS(el, rendered_less[src]); - } + var rendered = getRenderedLess(src); + if (rendered) { return void applyCSS(el, rendered); } var scope = opt.scope.attr('id') || 'cp-app-code-preview-content'; var scoped_src = '#' + scope + ' { ' + src + '}'; - console.error("RENDERING LESS", opt.cache); + //console.error("RENDERING LESS"); Less.render(scoped_src, {}, function (err, result) { - if (err) { return void console.error(err); } + // the console is the only feedback for users to know that they did something wrong + // but less rendering isn't intended so much as a feature but a useful tool to avoid + // leaking styles from the preview into the rest of the DOM. This is an improvement. + if (err) { + // we assume the compiler is deterministic. Something that returns an error once + // will do it again, so avoid successive calls by caching a truthy + // but non-functional string to block them. + rendered_less[src] = ' '; + return void console.error(err); + } var css = rendered_less[src] = result.css; applyCSS(el, css); }); @@ -523,8 +555,7 @@ define([ } }; - var cacheRenderedElement = function (cache, src, el) { // XXX - console.log("CACHING", cache, src, el); + var cacheRenderedElement = function (cache, src, el) { if (Array.isArray(cache[src])) { cache[src].push(el); } else { @@ -763,18 +794,20 @@ define([ if (target) { target.scrollIntoView(); } }); - $content.find('style').each(function (index, el) { // XXX + // transform style tags into pre tags with the same content + // to be handled by the less rendering plugin + $content.find('style').each(function (index, el) { var parent = el.parentElement; var pre = h('pre', { 'data-plugin': 'less', - 'less-src': el.innerText, + 'less-src': canonicalizeLess(el.innerText), style: 'display: none', }, el.innerText); parent.replaceChild(pre, el); }); // loop over plugin elements in the rendered content - $content.find('pre[data-plugin]').each(function (index, el) { // XXX + $content.find('pre[data-plugin]').each(function (index, el) { var type = el.getAttribute('data-plugin'); var plugin = plugins[type]; @@ -805,8 +838,7 @@ define([ if (typeof(cached) !== 'object') { try { plugin.render($el, { - scope: $content, // XXX - cache: plugin.cache, + scope: $content, }); } catch (e) { console.error(e); } return; From e5c8b6fd756675a3430697d221599bc4b71e6506 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 24 Jun 2021 15:08:09 +0530 Subject: [PATCH 15/79] WIP block remote images --- .../src/less2/include/markdown.less | 5 +- www/common/diffMarked.js | 89 ++++++++++++------- 2 files changed, 62 insertions(+), 32 deletions(-) diff --git a/customize.dist/src/less2/include/markdown.less b/customize.dist/src/less2/include/markdown.less index 43d9debaf..364b3790c 100644 --- a/customize.dist/src/less2/include/markdown.less +++ b/customize.dist/src/less2/include/markdown.less @@ -155,13 +155,14 @@ text-align: left; } +/* span.cp-inline-img-warning { - display: inline-block; + //display: inline-block; border: 1px solid red; a, br, strong { border: none; } - } + } */ //.cp-inline-img { } } diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index 61277bec2..4977cbbac 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -267,7 +267,21 @@ define([ return '
  • ' + text + '
    • \n'; }; - renderer.image = function (href, title, text) { + + var qualifiedHref = function (href) { + if (typeof(window.URL) === 'undefined') { return href; } + try { + var url = new URL(href, ApiConfig.httpUnsafeOrigin); + return url.href; + } catch (err) { + console.error(err); + return href; + } + }; + + var urlArgs = Util.find(ApiConfig, ['requireConf', 'urlArgs']) || ''; + + renderer.image = function (href, title, text) { // XXX if (href.slice(0,6) === '/file/') { // XXX this has been deprecated for about 3 years... use the same inline image handler as below? // DEPRECATED // Mediatag using markdown syntax should not be used anymore so they don't support @@ -285,13 +299,25 @@ define([ return mt; } - var img = h('img.cp-inline-img', { - src: href || '', - title: title || '', - alt: text || '', - }); - - return img.outerHTML; + var warning = h('span.cp-inline-img-warning', [ + h('img', { + src: '/images/broken.png?ver=' + ApiConfig.requireConf.urlArgs, + }), + h('br'), + h('span', { + //title: text, + }, "CryptPad blocked a remote image."), + h('br'), + h('a', { + href: qualifiedHref(href), + }, "Open its source in a new tab"), + h('br'), + h('a', { + href: 'https://docs.cryptpad.fr/en/user_guide/index.html?placeholder=remote_images', + }, 'learn why it was blocked'), + ]); + + return warning.outerHTML; }; restrictedRenderer.image = renderer.image; @@ -741,7 +767,11 @@ define([ if (typeof(patch) === 'string') { throw new Error(patch); } else { - DD.apply($content[0], patch); + try { + DD.apply($content[0], patch); + } catch (err) { + console.error(err); + } var $mts = $content.find('media-tag'); $mts.each(function (i, el) { var $mt = $(el).contextmenu(function (e) { @@ -797,32 +827,31 @@ define([ }); // replace remote images with links to those images - $content.find('img.cp-inline-img').each(function (index, el) { - var link = h('a', { - href: el.src, //common.getBounceURL(el.src), // XXX - target: '_blank', - rel: 'noopener noreferrer', - title: el.src, - }, [ - 'open image at ', - h('strong', el.src), - ]); + $content.find('span.cp-inline-img-warning').each(function (index, el) { // XXX +/* + var link = h('a', { + href: href, //el.src, //common.getBounceURL(el.src), // XXX + //target: '_blank', + //rel: 'noopener noreferrer', + //title: title, //el.src, + }, [ + 'open image at ', + h('strong', href), //el.src), + ]); +*/ + + + console.log('INLINE_IMG', index, el); + if (!el) { return; } + + var link = el.querySelector('a'); + if (!link) { return; } link.onclick = function (ev) { ev.preventDefault(); ev.stopPropagation(); - common.openURL(el.src); + common.openURL(link.href); }; - - var warning = h('span.cp-inline-img-warning', [ - "CryptPad disallows unencrypted images", - h('br'), - h('br'), - link, - ]); - - var parent = el.parentElement; - parent.replaceChild(warning, el); }); // transform style tags into pre tags with the same content From 4dcbddd174c13472b38c08801f4a396395a10b76 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 24 Jun 2021 17:33:45 +0530 Subject: [PATCH 16/79] WIP remote image styles --- .../src/less2/include/markdown.less | 26 ++++++++---- www/common/diffMarked.js | 42 ++++++------------- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/customize.dist/src/less2/include/markdown.less b/customize.dist/src/less2/include/markdown.less index 364b3790c..11c8a7eb4 100644 --- a/customize.dist/src/less2/include/markdown.less +++ b/customize.dist/src/less2/include/markdown.less @@ -155,15 +155,23 @@ text-align: left; } -/* - span.cp-inline-img-warning { - //display: inline-block; - border: 1px solid red; - a, br, strong { - border: none; - } - } */ - //.cp-inline-img { } + div.cp-inline-img-warning { + display: inline-block; + //border: 1px solid red !important; // @cp_markdown-border !important; + padding: 10px; + //color: @cryptpad_color_light_red; // very bad in light mode + //background: @cryptpad_color_red_fader; + + .cp-inline-img { + display: flex; + margin-bottom:10px; + } + .cp-alt-txt { + margin-left: 10px; + font-family: monospace; + font-size: 0.8em; + } + } } .markdown_cryptpad() { diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index 4977cbbac..d432b0841 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -279,8 +279,6 @@ define([ } }; - var urlArgs = Util.find(ApiConfig, ['requireConf', 'urlArgs']) || ''; - renderer.image = function (href, title, text) { // XXX if (href.slice(0,6) === '/file/') { // XXX this has been deprecated for about 3 years... use the same inline image handler as below? // DEPRECATED @@ -299,20 +297,22 @@ define([ return mt; } - var warning = h('span.cp-inline-img-warning', [ - h('img', { - src: '/images/broken.png?ver=' + ApiConfig.requireConf.urlArgs, - }), - h('br'), - h('span', { - //title: text, + var warning = h('div.cp-inline-img-warning', [ + h('div.cp-inline-img', [ + h('img.cp-inline-img', { + src: '/images/broken.png', + title: title || '', + }), + h('p.cp-alt-txt', text), + ]), + h('span.cp-img-block-notice', { }, "CryptPad blocked a remote image."), h('br'), - h('a', { + h('a.cp-remote-img', { href: qualifiedHref(href), }, "Open its source in a new tab"), h('br'), - h('a', { + h('a.cp-learn-more', { href: 'https://docs.cryptpad.fr/en/user_guide/index.html?placeholder=remote_images', }, 'learn why it was blocked'), ]); @@ -827,26 +827,10 @@ define([ }); // replace remote images with links to those images - $content.find('span.cp-inline-img-warning').each(function (index, el) { // XXX -/* - var link = h('a', { - href: href, //el.src, //common.getBounceURL(el.src), // XXX - //target: '_blank', - //rel: 'noopener noreferrer', - //title: title, //el.src, - }, [ - 'open image at ', - h('strong', href), //el.src), - ]); -*/ - - - console.log('INLINE_IMG', index, el); + $content.find('div.cp-inline-img-warning').each(function (index, el) { if (!el) { return; } - - var link = el.querySelector('a'); + var link = el.querySelector('a.cp-remote-img'); if (!link) { return; } - link.onclick = function (ev) { ev.preventDefault(); ev.stopPropagation(); From a86422266e08bfea731eb0b717ca5eae2e64a57a Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 24 Jun 2021 17:56:58 +0530 Subject: [PATCH 17/79] good contrast for most of the remote image warning --- customize.dist/src/less2/include/markdown.less | 7 ++++--- www/common/diffMarked.js | 16 ++++++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/customize.dist/src/less2/include/markdown.less b/customize.dist/src/less2/include/markdown.less index 11c8a7eb4..3e1837f96 100644 --- a/customize.dist/src/less2/include/markdown.less +++ b/customize.dist/src/less2/include/markdown.less @@ -157,10 +157,11 @@ div.cp-inline-img-warning { display: inline-block; - //border: 1px solid red !important; // @cp_markdown-border !important; padding: 10px; - //color: @cryptpad_color_light_red; // very bad in light mode - //background: @cryptpad_color_red_fader; + + color: @cryptpad_text_col; + background-color: @cryptpad_color_red_fader; + border: 1px solid @cryptpad_color_red; .cp-inline-img { display: flex; diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index d432b0841..f14b9aab6 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -279,8 +279,12 @@ define([ } }; - renderer.image = function (href, title, text) { // XXX - if (href.slice(0,6) === '/file/') { // XXX this has been deprecated for about 3 years... use the same inline image handler as below? + Messages.resources_imageBlocked = "CryptPad blocked a remote image"; // XXX + Messages.resources_openInNewTab = "Open its source in a new tab"; // XXX + Messages.resources_learnWhy = "Learn why it was blocked"; // XXX + + renderer.image = function (href, title, text) { + if (href.slice(0,6) === '/file/') { // XXX this has been deprecated for about 3 years. Maybe we should display a warning? // DEPRECATED // Mediatag using markdown syntax should not be used anymore so they don't support // password-protected files @@ -301,20 +305,20 @@ define([ h('div.cp-inline-img', [ h('img.cp-inline-img', { src: '/images/broken.png', - title: title || '', + //title: title || '', }), h('p.cp-alt-txt', text), ]), h('span.cp-img-block-notice', { - }, "CryptPad blocked a remote image."), + }, Messages.resources_imageBlocked), h('br'), h('a.cp-remote-img', { href: qualifiedHref(href), - }, "Open its source in a new tab"), + }, Messages.resources_openInNewTab), h('br'), h('a.cp-learn-more', { href: 'https://docs.cryptpad.fr/en/user_guide/index.html?placeholder=remote_images', - }, 'learn why it was blocked'), + }, Messages.resources_learnWhy), ]); return warning.outerHTML; From ec556ed261e3b56e7a160301e2d918835bd4b595 Mon Sep 17 00:00:00 2001 From: yflory Date: Thu, 24 Jun 2021 15:59:38 +0200 Subject: [PATCH 18/79] Export forms --- www/form/export.js | 60 ++++++++++++++++++++++++++++++++++++++++++++++ www/form/inner.js | 29 ++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 www/form/export.js diff --git a/www/form/export.js b/www/form/export.js new file mode 100644 index 000000000..e29c6dd87 --- /dev/null +++ b/www/form/export.js @@ -0,0 +1,60 @@ +define([ + '/common/common-util.js', + '/customize/messages.js' +], function (Util, Messages) { + var Export = {}; + + var escapeCSV = function (v) { + if (!/("|,)/.test(v)) { + return v || ''; + } + var value = ''; + var vv = (v || '').replaceAll('"', '""'); + value += '"' + vv + '"'; + return value; + }; + Export.results = function (content, answers, TYPES) { + console.log(content, answers, TYPES); + if (!content || !content.form) { return; } + var csv = ""; + var form = content.form; + + var questions = Object.keys(form).map(function (key) { + var obj = form[key]; + if (!obj) { return; } + return obj.q || Messages.form_default; + }).filter(Boolean); + questions.unshift(Messages.form_poll_time); // "Time" + questions.unshift(Messages.share_formView); // "Participant" + + questions.forEach(function (v, i) { + if (i) { csv += ','; } + csv += escapeCSV(v); + }); + + Object.keys(answers || {}).forEach(function (key) { + var obj = answers[key]; + csv += '\n'; + var time = new Date(obj.time).toISOString(); + var msg = obj.msg || {}; + var user = msg._userdata; + csv += escapeCSV(time); + csv += ',' + escapeCSV(user.name || Messages.anonymous); + Object.keys(form).forEach(function (key) { + csv += ',' + escapeCSV(String(msg[key])); + }); + }); + console.log(csv); + return csv; + }; + + Export.main = function (content, cb) { + var json = Util.clone(content || {}); + delete json.answers; + cb(new Blob([JSON.stringify(json, 0, 2)], { + type: 'application/json;charset=utf-8' + })); + }; + + return Export; +}); diff --git a/www/form/inner.js b/www/form/inner.js index 80c50a0d6..123e86d5b 100644 --- a/www/form/inner.js +++ b/www/form/inner.js @@ -4,6 +4,7 @@ define([ '/bower_components/chainpad-crypto/crypto.js', '/common/sframe-app-framework.js', '/common/toolbar.js', + '/form/export.js', '/bower_components/nthen/index.js', '/common/sframe-common.js', '/common/common-util.js', @@ -30,6 +31,8 @@ define([ 'cm/mode/gfm/gfm', 'css!cm/lib/codemirror.css', + '/bower_components/file-saver/FileSaver.min.js', + 'css!/bower_components/codemirror/lib/codemirror.css', 'css!/bower_components/codemirror/addon/dialog/dialog.css', 'css!/bower_components/codemirror/addon/fold/foldgutter.css', @@ -42,6 +45,7 @@ define([ Crypto, Framework, Toolbar, + Exporter, nThen, SFCommon, Util, @@ -1656,9 +1660,22 @@ define([ var controls = h('div.cp-form-creator-results-controls'); var $controls = $(controls).appendTo($container); + Messages.form_exportCSV = "Export results as CSV"; + var exportButton = h('button.btn.btn-secondary', Messages.form_exportCSV); + var exportCSV = h('div.cp-form-creator-results-export', exportButton); + $(exportCSV).appendTo($container); var results = h('div.cp-form-creator-results-content'); var $results = $(results).appendTo($container); + $(exportButton).click(function () { + var csv = Exporter.results(content, answers, TYPES); + if (!csv) { return void UI.warn(Messages.error); } + var suggestion = APP.framework._.title.suggestTitle('cryptpad-document'); + var title = Util.fixFileName(suggestion) + '.csv'; + window.saveAs(new Blob([csv], { + type: 'text/csv' + }), title); + }); var summary = true; var form = content.form; @@ -2315,6 +2332,7 @@ define([ var andThen = function (framework) { framework.start(); + APP.framework = framework; var evOnChange = Util.mkEvent(); var content = {}; @@ -2739,6 +2757,17 @@ define([ return content; }); + framework.setFileImporter({ accept: ['.json'] }, function (newContent) { + var parsed = JSON.parse(newContent || {}); + parsed.answers = content.answers; + return parsed; + }); + + framework.setFileExporter(['.json'], function(cb, ext) { + Exporter.main(content, cb, ext); + }, true); + + }; Framework.create({ From ef885eed5c20b60810d1cdfbd16e85e26dfa0f5b Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 25 Jun 2021 11:51:18 +0530 Subject: [PATCH 19/79] leave notes for finalizing remote image warning --- www/common/diffMarked.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index f14b9aab6..9e40ad3db 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -305,7 +305,7 @@ define([ h('div.cp-inline-img', [ h('img.cp-inline-img', { src: '/images/broken.png', - //title: title || '', + //title: title || '', // XXX sort out tippy issues (double-title) }), h('p.cp-alt-txt', text), ]), @@ -317,7 +317,7 @@ define([ }, Messages.resources_openInNewTab), h('br'), h('a.cp-learn-more', { - href: 'https://docs.cryptpad.fr/en/user_guide/index.html?placeholder=remote_images', + href: 'https://docs.cryptpad.fr/en/user_guide/index.html?placeholder=remote_images', // XXX point to an actual page }, Messages.resources_learnWhy), ]); From 4a147815f6b7cdd13bd5b74ad90108f5bf315878 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 25 Jun 2021 11:52:24 +0530 Subject: [PATCH 20/79] disable server_tokens test until an easy solution is in place --- www/checkup/main.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/www/checkup/main.js b/www/checkup/main.js index 2639ff156..732197a2d 100644 --- a/www/checkup/main.js +++ b/www/checkup/main.js @@ -732,6 +732,7 @@ define([ cb(isHTTPS(trimmedUnsafe) && isHTTPS(trimmedSafe)); }); +/* assert(function (cb, msg) { setWarningClass(msg); $.ajax(cacheBuster('/'), { @@ -756,8 +757,7 @@ define([ " header. This information can make it easier for attackers to find and exploit known vulnerabilities. ", ]; - - if (family === 'NGINX') { + if (family === 'NGINX') { // XXX incorrect instructions for HTTP2. needs a recompile? msg.appendChild(h('span', text.concat([ "This can be addressed by setting ", code("server_tokens off"), @@ -779,6 +779,7 @@ define([ } }); }); +*/ if (false) { assert(function (cb, msg) { From 9104441eb2a6d8c88ad236ce3e8c68b6ec908f45 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 25 Jun 2021 11:58:05 +0530 Subject: [PATCH 21/79] prototype instance purpose radio selection --- www/admin/app-admin.less | 11 ++++++++ www/admin/inner.js | 55 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/www/admin/app-admin.less b/www/admin/app-admin.less index 17450361c..227792540 100644 --- a/www/admin/app-admin.less +++ b/www/admin/app-admin.less @@ -203,6 +203,17 @@ } } + .cp-admin-radio-container { + display: flex; + align-items: left; //center; + flex-wrap: wrap; + flex-direction: column; + label { + margin-right: 40px; + margin-top: 5px; + } + } + .cp-admin-broadcast-form { input.flatpickr-input { width: 307.875px !important; // same width as flatpickr calendar diff --git a/www/admin/inner.js b/www/admin/inner.js index f8bcad681..883db0f84 100644 --- a/www/admin/inner.js +++ b/www/admin/inner.js @@ -94,6 +94,7 @@ define([ 'cp-admin-list-my-instance', 'cp-admin-consent-to-contact', 'cp-admin-remove-donate-button', + 'cp-admin-instance-purpose', // XXX ], }; @@ -1853,6 +1854,60 @@ define([ }, }); + Messages.admin_instancePurposeTitle = "Instance purpose"; // XXX + Messages.admin_instancePurposeHint = "Why do you run this instance? Your answer will be shared with the developers unless you have disabled telemetry"; // XXX + + Messages.admin_purpose_noanswer = "I prefer not to answer"; // XXX + Messages.admin_purpose_experiment = "To test the CryptPad platform"; // XXX + Messages.admin_purpose_development = "To develop new features for CryptPad"; // XXX + + Messages.admin_purpose_personal = "For myself, family, and friends"; // XXX + + Messages.admin_purpose_business = "For my business's external use"; // XXX + Messages.admin_purpose_intranet = "For my business's internal use"; // XXX + + Messages.admin_purpose_school = "For my school, college, or university"; // XXX + Messages.admin_purpose_org = "For a non-profit organization or advocacy group"; // XXX + + Messages.admin_purpose_commercial = "To provide a commercial service"; // XXX + Messages.admin_purpose_public = "To provide a free service"; // XXX + + create['instance-purpose'] = function () { + var key = 'instance-purpose'; + var $div = makeBlock(key); + + var values = [ + 'noanswer', + 'development', + 'experiment', + 'personal', + 'business', // as a public resource for my business clients + 'intranet', // for my business's _internal use_ + 'school', + 'org', + 'commercial', + 'public', // to provide a free service (for the public) + ]; + + var defaultPurpose = 'noanswer'; + var purpose = defaultPurpose; + + var opts = h('div.cp-admin-radio-container', [ + values.map(function (key) { + var full_key = 'admin_purpose_' + key; + return UI.createRadio('cp-instance-purpose-radio', 'cp-instance-purpose-radio-'+key, + Messages[full_key] || Messages._getKey(full_key, [defaultPurpose]), + key === purpose, { + input: { value: key }, + label: { class: 'noTitle' } + }); + }) + ]); + + $div.append(opts); + return $div; + }; + var hideCategories = function () { APP.$rightside.find('> div').hide(); }; From 0b8239241b5bd09a02eb3a8af0c55d7d233b5892 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 25 Jun 2021 12:23:46 +0530 Subject: [PATCH 22/79] integrate styles for remote images --- .../src/less2/include/markdown.less | 22 ++++++++++++++++--- www/common/diffMarked.js | 10 +++++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/customize.dist/src/less2/include/markdown.less b/customize.dist/src/less2/include/markdown.less index 3e1837f96..f52b2666f 100644 --- a/customize.dist/src/less2/include/markdown.less +++ b/customize.dist/src/less2/include/markdown.less @@ -156,21 +156,37 @@ } div.cp-inline-img-warning { + @cryptpad_test_red_fader: fade(@cryptpad_color_red, 15%); // XXX display: inline-block; padding: 10px; - color: @cryptpad_text_col; - background-color: @cryptpad_color_red_fader; + color: @cryptpad_text_col; // XXX + background-color: @cryptpad_test_red_fader; // XXX @cryptpad_color_red_fader; border: 1px solid @cryptpad_color_red; .cp-inline-img { display: flex; - margin-bottom:10px; + margin-bottom: 10px; } .cp-alt-txt { margin-left: 10px; font-family: monospace; font-size: 0.8em; + color: fade(@cryptpad_text_col, 90%); + } + a { + color: @cryptpad_text_col; + font-size: 0.8em; + &.cp-remote-img::before { + font-family: FontAwesome; + //content: "\f08e\00a0"; + content: "\f08e\00a0\00a0"; + } + &.cp-learn-more::before { + font-family: FontAwesome; + content: "\f059\00a0"; + //content: "\f059\00a0\00a0"; + } } } } diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index 9e40ad3db..1f4fadefb 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -314,11 +314,17 @@ define([ h('br'), h('a.cp-remote-img', { href: qualifiedHref(href), - }, Messages.resources_openInNewTab), + }, [ + //h('i.fa.fa-external-link'), // XXX + Messages.resources_openInNewTab + ]), h('br'), h('a.cp-learn-more', { href: 'https://docs.cryptpad.fr/en/user_guide/index.html?placeholder=remote_images', // XXX point to an actual page - }, Messages.resources_learnWhy), + }, [ + //h('i.fa.fa-question-circle'), // XXX + Messages.resources_learnWhy + ]), ]); return warning.outerHTML; From 9027409ce5f53de323b41ee669cfd1d89323dbe9 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 25 Jun 2021 12:29:13 +0530 Subject: [PATCH 23/79] serverside components of instancePurpose flag --- lib/decrees.js | 3 +++ lib/stats.js | 2 ++ 2 files changed, 5 insertions(+) diff --git a/lib/decrees.js b/lib/decrees.js index 960ece1ef..5f599705e 100644 --- a/lib/decrees.js +++ b/lib/decrees.js @@ -173,6 +173,9 @@ commands.SET_SUPPORT_MAILBOX = makeGenericSetter('supportMailbox', function (arg return args_isString(args) && Core.isValidPublicKey(args[0]); }); +// CryptPad_AsyncStore.rpc.send('ADMIN', [ 'ADMIN_DECREE', ['SET_INSTANCE_PURPOSE', ["development"]]], console.log) +commands.SET_INSTANCE_PURPOSE = makeGenericSetter('instancePurpose', args_isString); + // Maintenance: Empty string or an object with a start and end time var isNumber = function (value) { return typeof(value) === "number" && !isNaN(value); diff --git a/lib/stats.js b/lib/stats.js index da820f7b8..2a513da44 100644 --- a/lib/stats.js +++ b/lib/stats.js @@ -14,6 +14,8 @@ Stats.instanceData = function (Env) { adminEmail: Env.adminEmail, consentToContact: Boolean(Env.consentToContact), + + instancePurpose: Env.instancePurpose, // XXX }; /* We reserve the right to choose not to include instances From ad493e5049c14f403553be2106056a61380e0306 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 25 Jun 2021 12:29:54 +0530 Subject: [PATCH 24/79] add WIP features to changelog --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d33d0e25c..b41b1d9b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +# WIP + +* WIP file conversion utilities +* server + * `installMethod: 'unspecified'` in the default config to distinguish docker installs + * `instancePurpose` on admin panel +* display warnings when remote resources are blocked + * in code preview +* restrict style tags to a scope when rendering them in markdown preview by compiling their content as scoped less +* iPhone/Safari calendar and notification fixes (data parsing errors) +* checkup + * display actual FLoC header in checkup test + * WIP check for `server_tokens` settings (needs work for HTTP2) + * nicer output in error/warning tables +* form templates +* guard against a type error in `getAccessKeys` +* guard against invalid or malicious input when constructing media-tags for embedding in markdown + # 4.7.0 ## Goals From 3cbf4c9d6f5a7fa5b7b8991173fe79e5ca088181 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 25 Jun 2021 13:04:29 +0530 Subject: [PATCH 25/79] save instancePurpose choice to server from admin panel --- lib/commands/admin-rpc.js | 1 + www/admin/inner.js | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/lib/commands/admin-rpc.js b/lib/commands/admin-rpc.js index ab48a1086..59750571e 100644 --- a/lib/commands/admin-rpc.js +++ b/lib/commands/admin-rpc.js @@ -326,6 +326,7 @@ var instanceStatus = function (Env, Server, cb) { blockDailyCheck: Env.blockDailyCheck, updateAvailable: Env.updateAvailable, + instancePurpose: Env.instancePurpose, }); }; diff --git a/www/admin/inner.js b/www/admin/inner.js index 883db0f84..592c77a5f 100644 --- a/www/admin/inner.js +++ b/www/admin/inner.js @@ -1861,7 +1861,7 @@ define([ Messages.admin_purpose_experiment = "To test the CryptPad platform"; // XXX Messages.admin_purpose_development = "To develop new features for CryptPad"; // XXX - Messages.admin_purpose_personal = "For myself, family, and friends"; // XXX + Messages.admin_purpose_personal = "For myself, family, or friends"; // XXX Messages.admin_purpose_business = "For my business's external use"; // XXX Messages.admin_purpose_intranet = "For my business's internal use"; // XXX @@ -1872,6 +1872,13 @@ define([ Messages.admin_purpose_commercial = "To provide a commercial service"; // XXX Messages.admin_purpose_public = "To provide a free service"; // XXX + var sendDecree = function (data, cb) { + sFrameChan.query('Q_ADMIN_RPC', { + cmd: 'ADMIN_DECREE', + data: data, + }, cb); + }; + create['instance-purpose'] = function () { var key = 'instance-purpose'; var $div = makeBlock(key); @@ -1890,7 +1897,7 @@ define([ ]; var defaultPurpose = 'noanswer'; - var purpose = defaultPurpose; + var purpose = APP.instanceStatus.instancePurpose || defaultPurpose; var opts = h('div.cp-admin-radio-container', [ values.map(function (key) { @@ -1904,7 +1911,35 @@ define([ }) ]); + var $opts = $(opts); + //var $br = $(h('br',)); + //$div.append($br); + $div.append(opts); + + var setPurpose = function (value, cb) { + sendDecree([ + 'SET_INSTANCE_PURPOSE', + [ value] + ], cb); + }; + //var spinner = UI.makeSpinner($br); // XXX + + $opts.on('change', function () { + var val = $opts.find('input:radio:checked').val(); + console.log(val); + //spinner.spin(); + setPurpose(val, function (e, response) { + if (e || response.error) { + UI.warn(Messages.error); + //spinner.hide(); + return; + } + //spinner.done(); + UI.log(Messages.saved); + }); + }); + return $div; }; From caece0123e9b0c13228a88513d9843557fb593b8 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 25 Jun 2021 20:22:16 +0530 Subject: [PATCH 26/79] archive pin logs instead of removing them outright --- lib/commands/pin-rpc.js | 4 ++-- lib/historyKeeper.js | 2 ++ lib/storage/file.js | 9 ++++++--- lib/workers/db-worker.js | 3 +++ scripts/evict-archived.js | 2 ++ scripts/evict-inactive.js | 2 ++ scripts/lint-translations.js | 8 ++++++++ 7 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 scripts/lint-translations.js diff --git a/lib/commands/pin-rpc.js b/lib/commands/pin-rpc.js index 93d922620..ff6bfdbac 100644 --- a/lib/commands/pin-rpc.js +++ b/lib/commands/pin-rpc.js @@ -115,8 +115,8 @@ Pinning.getTotalSize = function (Env, safeKey, cb) { */ Pinning.removePins = function (Env, safeKey, cb) { // FIXME respect the queue - Env.pinStore.removeChannel(safeKey, function (err) { - Env.Log.info('DELETION_PIN_BY_OWNER_RPC', { + Env.pinStore.archiveChannel(safeKey, function (err) { + Env.Log.info('ARCHIVAL_PIN_BY_OWNER_RPC', { safeKey: safeKey, status: err? String(err): 'SUCCESS', }); diff --git a/lib/historyKeeper.js b/lib/historyKeeper.js index daa041b5f..44afe0ae0 100644 --- a/lib/historyKeeper.js +++ b/lib/historyKeeper.js @@ -123,6 +123,8 @@ module.exports.create = function (Env, cb) { Store.create({ filePath: pinPath, archivePath: Env.paths.archive, + // indicate that archives should be put in a 'pins' archvie folder + volumeId: 'pins', }, w(function (err, s) { if (err) { throw err; } Env.pinStore = s; diff --git a/lib/storage/file.js b/lib/storage/file.js index 825f14066..30e6c7644 100644 --- a/lib/storage/file.js +++ b/lib/storage/file.js @@ -51,7 +51,7 @@ var mkPath = function (env, channelId) { }; var mkArchivePath = function (env, channelId) { - return Path.join(env.archiveRoot, 'datastore', channelId.slice(0, 2), channelId) + '.ndjson'; + return Path.join(env.archiveRoot, env.volumeId, channelId.slice(0, 2), channelId) + '.ndjson'; }; var mkMetadataPath = function (env, channelId) { @@ -59,7 +59,7 @@ var mkMetadataPath = function (env, channelId) { }; var mkArchiveMetadataPath = function (env, channelId) { - return Path.join(env.archiveRoot, 'datastore', channelId.slice(0, 2), channelId) + '.metadata.ndjson'; + return Path.join(env.archiveRoot, env.volumeId, channelId.slice(0, 2), channelId) + '.metadata.ndjson'; }; var mkTempPath = function (env, channelId) { @@ -1044,6 +1044,9 @@ module.exports.create = function (conf, _cb) { var env = { root: conf.filePath || './datastore', archiveRoot: conf.archivePath || './data/archive', + // supply a volumeId if you want a store to archive channels to and from + // to its own subpath within the archive directory + volumeId: conf.volumeId || 'datastore', channels: { }, batchGetChannel: BatchRead('store_batch_channel'), }; @@ -1076,7 +1079,7 @@ module.exports.create = function (conf, _cb) { } })); // make sure the cold storage directory exists - Fse.mkdirp(env.archiveRoot, PERMISSIVE, w(function (err) { + Fse.mkdirp(Path.join(env.archiveRoot, env.volumeId), PERMISSIVE, w(function (err) { if (err && err.code !== 'EEXIST') { w.abort(); return void cb(err); diff --git a/lib/workers/db-worker.js b/lib/workers/db-worker.js index 8585cc3f6..137b756b4 100644 --- a/lib/workers/db-worker.js +++ b/lib/workers/db-worker.js @@ -66,6 +66,9 @@ const init = function (config, _cb) { Store.create({ filePath: config.pinPath, archivePath: config.archivePath, + // important to initialize the pinstore with its own volume id + // otherwise archived pin logs will get mixed in with channels + volumeId: 'pins', }, w(function (err, _pinStore) { if (err) { w.abort(); diff --git a/scripts/evict-archived.js b/scripts/evict-archived.js index 7f90f9ff5..255246e99 100644 --- a/scripts/evict-archived.js +++ b/scripts/evict-archived.js @@ -56,6 +56,8 @@ var prepareEnv = function (Env, cb) { Store.create({ filePath: config.pinPath, + // archive pin logs to their own subpath + volumeId: 'pins', }, w(function (err, _) { if (err) { w.abort(); diff --git a/scripts/evict-inactive.js b/scripts/evict-inactive.js index bf7e1ca5b..2521e014f 100644 --- a/scripts/evict-inactive.js +++ b/scripts/evict-inactive.js @@ -56,6 +56,8 @@ var prepareEnv = function (Env, cb) { Store.create({ filePath: config.pinPath, + // archive pin logs to their own subpath + volumeId: 'pins', }, w(function (err, _) { if (err) { w.abort(); diff --git a/scripts/lint-translations.js b/scripts/lint-translations.js new file mode 100644 index 000000000..a38cd615a --- /dev/null +++ b/scripts/lint-translations.js @@ -0,0 +1,8 @@ +// TODO unify the following scripts + // unused-translations.js + // find-html-translations + +// more linting + // Search for 'Cryptpad' string (should be 'CryptPad') + // Search English for -ise\s + From 9806d718d5aaffee95c74d346ae12a88e1ed50d4 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 25 Jun 2021 20:53:09 +0530 Subject: [PATCH 27/79] implement block archival --- lib/commands/block.js | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/lib/commands/block.js b/lib/commands/block.js index 8180cb68e..e405ede85 100644 --- a/lib/commands/block.js +++ b/lib/commands/block.js @@ -86,6 +86,20 @@ var createLoginBlockPath = function (Env, publicKey) { // FIXME BLOCKS return Path.join(Env.paths.block, safeKey.slice(0, 2), safeKey); }; +var createLoginBlockArchivePath = function (Env, publicKey) { + // prepare publicKey to be used as a file name + var safeKey = Util.escapeKeyCharacters(publicKey); + + // validate safeKey + if (typeof(safeKey) !== 'string') { + return; + } + + // derive the full path + // /home/cryptpad/cryptpad/block/fg/fg32kefksjdgjkewrjksdfksjdfsdfskdjfsfd + return Path.join(Env.paths.archive, 'block', safeKey.slice(0, 2), safeKey); +}; + Block.validateAncestorProof = function (Env, proof, _cb) { var cb = Util.once(Util.mkAsync(_cb)); /* prove that you own an existing block by signing for its publicKey */ @@ -216,7 +230,7 @@ Block.writeLoginBlock = function (Env, safeKey, msg, _cb) { // FIXME BLOCKS information, we can just sign some constant and use that as proof. */ -Block.removeLoginBlock = function (Env, safeKey, msg, cb) { // FIXME BLOCKS +Block.removeLoginBlock = function (Env, safeKey, msg, cb) { var publicKey = msg[0]; var signature = msg[1]; var block = Nacl.util.decodeUTF8('DELETE_BLOCK'); // clients and the server will have to agree on this constant @@ -230,21 +244,28 @@ Block.removeLoginBlock = function (Env, safeKey, msg, cb) { // FIXME BLOCKS validateLoginBlock(Env, publicKey, signature, block, function (e /*::, validatedBlock */) { if (e) { return void cb(e); } // derive the filepath - var path = createLoginBlockPath(Env, publicKey); + var currentPath = createLoginBlockPath(Env, publicKey); // make sure the path is valid - if (typeof(path) !== 'string') { + if (typeof(currentPath) !== 'string') { return void cb('E_INVALID_BLOCK_PATH'); } - // FIXME COLDSTORAGE - Fs.unlink(path, function (err) { - Env.Log.info('DELETION_BLOCK_BY_OWNER_RPC', { + var archivePath = createLoginBlockArchivePath(Env, publicKey); + // make sure the path is valid + if (typeof(archivePath) !== 'string') { + return void cb('E_INVALID_BLOCK_ARCHIVAL_PATH'); + } + + Fse.move(currentPath, archivePath, { + overwrite: true, + }, function (err) { + Env.Log.info('ARCHIVAL_BLOCK_BY_OWNER_RPC', { publicKey: publicKey, - path: path, + currentPath: currentPath, + archivePath: archivePath, status: err? String(err): 'SUCCESS', }); - if (err) { return void cb(err); } cb(); }); From ba1a7b37e1a603b55dd6c979689ec481c9877d29 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 28 Jun 2021 14:39:26 +0530 Subject: [PATCH 28/79] separate validation and storage methods for blocks --- lib/commands/block.js | 117 ++++++++---------------------------------- lib/storage/block.js | 92 +++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 96 deletions(-) create mode 100644 lib/storage/block.js diff --git a/lib/commands/block.js b/lib/commands/block.js index e405ede85..947740908 100644 --- a/lib/commands/block.js +++ b/lib/commands/block.js @@ -1,14 +1,10 @@ /*jshint esversion: 6 */ /* globals Buffer*/ -var Block = module.exports; - -const Fs = require("fs"); -const Fse = require("fs-extra"); -const Path = require("path"); +const Block = module.exports; const Nacl = require("tweetnacl/nacl-fast"); const nThen = require("nthen"); - const Util = require("../common-util"); +const BlockStore = require("../storage/block"); /* We assume that the server is secured against MitM attacks @@ -31,7 +27,9 @@ const Util = require("../common-util"); author of the block, since we assume that the block will have been encrypted with xsalsa20-poly1305 which is authenticated. */ -var validateLoginBlock = function (Env, publicKey, signature, block, cb) { // FIXME BLOCKS +var validateLoginBlock = function (Env, publicKey, signature, block, _cb) { // FIXME BLOCKS + var cb = Util.once(Util.mkAsync(_cb)); + // convert the public key to a Uint8Array and validate it if (typeof(publicKey) !== 'string') { return void cb('E_INVALID_KEY'); } @@ -72,34 +70,6 @@ var validateLoginBlock = function (Env, publicKey, signature, block, cb) { // FI return void cb(null, u8_block); }; -var createLoginBlockPath = function (Env, publicKey) { // FIXME BLOCKS - // prepare publicKey to be used as a file name - var safeKey = Util.escapeKeyCharacters(publicKey); - - // validate safeKey - if (typeof(safeKey) !== 'string') { - return; - } - - // derive the full path - // /home/cryptpad/cryptpad/block/fg/fg32kefksjdgjkewrjksdfksjdfsdfskdjfsfd - return Path.join(Env.paths.block, safeKey.slice(0, 2), safeKey); -}; - -var createLoginBlockArchivePath = function (Env, publicKey) { - // prepare publicKey to be used as a file name - var safeKey = Util.escapeKeyCharacters(publicKey); - - // validate safeKey - if (typeof(safeKey) !== 'string') { - return; - } - - // derive the full path - // /home/cryptpad/cryptpad/block/fg/fg32kefksjdgjkewrjksdfksjdfsdfskdjfsfd - return Path.join(Env.paths.archive, 'block', safeKey.slice(0, 2), safeKey); -}; - Block.validateAncestorProof = function (Env, proof, _cb) { var cb = Util.once(Util.mkAsync(_cb)); /* prove that you own an existing block by signing for its publicKey */ @@ -117,31 +87,26 @@ Block.validateAncestorProof = function (Env, proof, _cb) { return void cb('E_INVALID_ANCESTOR_PROOF'); } // else fall through to next step - }).nThen(function (w) { - var path = createLoginBlockPath(Env, pub); - Fs.access(path, Fs.constants.F_OK, w(function (err) { - if (!err) { return; } - w.abort(); // else - return void cb("E_MISSING_ANCESTOR"); - })); }).nThen(function () { - cb(void 0, pub); + Block.check(Env, pub, function (err) { + if (err) { return void cb('E_MISSING_ANCESTOR'); } + cb(void 0, pub); + }); }); } catch (err) { return void cb(err); } }; -Block.writeLoginBlock = function (Env, safeKey, msg, _cb) { // FIXME BLOCKS +Block.writeLoginBlock = function (Env, safeKey, msg, _cb) { var cb = Util.once(Util.mkAsync(_cb)); - //console.log(msg); var publicKey = msg[0]; var signature = msg[1]; var block = msg[2]; var registrationProof = msg[3]; var previousKey; - var validatedBlock, parsed, path; + var validatedBlock, path; nThen(function (w) { if (Util.escapeKeyCharacters(publicKey) !== safeKey) { w.abort(); @@ -181,33 +146,9 @@ Block.writeLoginBlock = function (Env, safeKey, msg, _cb) { // FIXME BLOCKS } validatedBlock = _validatedBlock; - - // derive the filepath - path = createLoginBlockPath(Env, publicKey); - - // make sure the path is valid - if (typeof(path) !== 'string') { - return void cb('E_INVALID_BLOCK_PATH'); - } - - parsed = Path.parse(path); - if (!parsed || typeof(parsed.dir) !== 'string') { - w.abort(); - return void cb("E_INVALID_BLOCK_PATH_2"); - } - })); - }).nThen(function (w) { - // make sure the path to the file exists - Fse.mkdirp(parsed.dir, w(function (e) { - if (e) { - w.abort(); - cb(e); - } })); }).nThen(function () { - // actually write the block - Fs.writeFile(path, Buffer.from(validatedBlock), { encoding: "binary", }, function (err) { - if (err) { return void cb(err); } + BlockStore.write(Env, publicKey, Buffer.from(validatedBlock), function (err) { Env.Log.info('BLOCK_WRITE_BY_OWNER', { safeKey: safeKey, blockId: publicKey, @@ -215,11 +156,13 @@ Block.writeLoginBlock = function (Env, safeKey, msg, _cb) { // FIXME BLOCKS previousKey: previousKey, path: path, }); - cb(); + cb(err); }); }); }; +const DELETE_BLOCK = Nacl.util.decodeUTF8('DELETE_BLOCK'); + /* When users write a block, they upload the block, and provide a signature proving that they deserve to be able to write to @@ -230,10 +173,11 @@ Block.writeLoginBlock = function (Env, safeKey, msg, _cb) { // FIXME BLOCKS information, we can just sign some constant and use that as proof. */ -Block.removeLoginBlock = function (Env, safeKey, msg, cb) { +Block.removeLoginBlock = function (Env, safeKey, msg, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + var publicKey = msg[0]; var signature = msg[1]; - var block = Nacl.util.decodeUTF8('DELETE_BLOCK'); // clients and the server will have to agree on this constant nThen(function (w) { if (Util.escapeKeyCharacters(publicKey) !== safeKey) { @@ -241,33 +185,14 @@ Block.removeLoginBlock = function (Env, safeKey, msg, cb) { return void cb("INCORRECT_KEY"); } }).nThen(function () { - validateLoginBlock(Env, publicKey, signature, block, function (e /*::, validatedBlock */) { + validateLoginBlock(Env, publicKey, signature, DELETE_BLOCK, function (e /*::, validatedBlock */) { if (e) { return void cb(e); } - // derive the filepath - var currentPath = createLoginBlockPath(Env, publicKey); - - // make sure the path is valid - if (typeof(currentPath) !== 'string') { - return void cb('E_INVALID_BLOCK_PATH'); - } - - var archivePath = createLoginBlockArchivePath(Env, publicKey); - // make sure the path is valid - if (typeof(archivePath) !== 'string') { - return void cb('E_INVALID_BLOCK_ARCHIVAL_PATH'); - } - - Fse.move(currentPath, archivePath, { - overwrite: true, - }, function (err) { + BlockStore.archive(Env, publicKey, function (err) { Env.Log.info('ARCHIVAL_BLOCK_BY_OWNER_RPC', { publicKey: publicKey, - currentPath: currentPath, - archivePath: archivePath, status: err? String(err): 'SUCCESS', }); - if (err) { return void cb(err); } - cb(); + cb(err); }); }); }); diff --git a/lib/storage/block.js b/lib/storage/block.js new file mode 100644 index 000000000..5dfc393a5 --- /dev/null +++ b/lib/storage/block.js @@ -0,0 +1,92 @@ +/*jshint esversion: 6 */ +const Block = module.exports; +const Util = require("../common-util"); +const Path = require("path"); +const Fs = require("fs"); +const Fse = require("fs-extra"); +const nThen = require("nthen"); + +Block.mkPath = function (Env, publicKey) { + // prepare publicKey to be used as a file name + var safeKey = Util.escapeKeyCharacters(publicKey); + + // validate safeKey + if (typeof(safeKey) !== 'string') { return; } + + // derive the full path + // /home/cryptpad/cryptpad/block/fg/fg32kefksjdgjkewrjksdfksjdfsdfskdjfsfd + return Path.join(Env.paths.block, safeKey.slice(0, 2), safeKey); +}; + +Block.mkArchivePath = function (Env, publicKey) { + // prepare publicKey to be used as a file name + var safeKey = Util.escapeKeyCharacters(publicKey); + + // validate safeKey + if (typeof(safeKey) !== 'string') { + return; + } + + // derive the full path + // /home/cryptpad/cryptpad/block/fg/fg32kefksjdgjkewrjksdfksjdfsdfskdjfsfd + return Path.join(Env.paths.archive, 'block', safeKey.slice(0, 2), safeKey); +}; + +Block.archive = function (Env, publicKey, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + + // derive the filepath + var currentPath = Block.mkPath(Env, publicKey); + + // make sure the path is valid + if (typeof(currentPath) !== 'string') { + return void cb('E_INVALID_BLOCK_PATH'); + } + + var archivePath = Block.mkArchivePath(Env, publicKey); + // make sure the path is valid + if (typeof(archivePath) !== 'string') { + return void cb('E_INVALID_BLOCK_ARCHIVAL_PATH'); + } + + Fse.move(currentPath, archivePath, { + overwrite: true, + }, cb); +}; + +Block.check = function (Env, publicKey, _cb) { // 'check' because 'exists' implies boolean + var cb = Util.once(Util.mkAsync(_cb)); + var path = Block.mkPath(Env, publicKey); + Fs.access(path, Fs.constants.F_OK, cb); +}; + +Block.write = function (Env, publicKey, buffer, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + var path = Block.mkPath(Env, publicKey); + if (typeof(path) !== 'string') { return void cb('INVALID_PATH'); } + var parsed = Path.parse(path); + + nThen(function (w) { + Fse.mkdirp(parsed.dir, w(function (err) { + if (!err) { return; } + w.abort(); + cb(err); + })); + }).nThen(function () { + // XXX BLOCK check whether this overwrites a block + // XXX archive the old one if so + Fs.writeFile(path, buffer, { encoding: 'binary' }, cb); + }); +}; + +/* +Block.create = function (opt, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + + var env = { + root: opt.root || '', // XXX + + + }; +}; +*/ From 7c7acbeae68580254d8a953393f751d2a1689a8c Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 28 Jun 2021 15:07:48 +0530 Subject: [PATCH 29/79] delegate block validation to workers --- lib/commands/block.js | 20 +++++++++++++------- lib/workers/db-worker.js | 4 ++++ lib/workers/index.js | 9 +++++++++ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/lib/commands/block.js b/lib/commands/block.js index 947740908..ea6a19dc2 100644 --- a/lib/commands/block.js +++ b/lib/commands/block.js @@ -27,7 +27,7 @@ const BlockStore = require("../storage/block"); author of the block, since we assume that the block will have been encrypted with xsalsa20-poly1305 which is authenticated. */ -var validateLoginBlock = function (Env, publicKey, signature, block, _cb) { // FIXME BLOCKS +Block.validateLoginBlock = function (Env, publicKey, signature, block, _cb) { var cb = Util.once(Util.mkAsync(_cb)); // convert the public key to a Uint8Array and validate it @@ -67,7 +67,7 @@ var validateLoginBlock = function (Env, publicKey, signature, block, _cb) { // F // call back with (err) if unsuccessful if (!verified) { return void cb("E_COULD_NOT_VERIFY"); } - return void cb(null, u8_block); + return void cb(null, block); }; Block.validateAncestorProof = function (Env, proof, _cb) { @@ -135,20 +135,26 @@ Block.writeLoginBlock = function (Env, safeKey, msg, _cb) { previousKey = provenKey; })); }).nThen(function (w) { - validateLoginBlock(Env, publicKey, signature, block, w(function (e, _validatedBlock) { + Env.validateLoginBlock(publicKey, signature, block, w(function (e, _validatedBlock) { if (e) { w.abort(); return void cb(e); } - if (!(_validatedBlock instanceof Uint8Array)) { + if (typeof(_validatedBlock) !== 'string') { w.abort(); - return void cb('E_INVALID_BLOCK'); + return void cb('E_INVALID_BLOCK_RETURNED'); } validatedBlock = _validatedBlock; })); }).nThen(function () { - BlockStore.write(Env, publicKey, Buffer.from(validatedBlock), function (err) { + var buffer; + try { + buffer = Buffer.from(Nacl.util.decodeBase64(validatedBlock)); + } catch (err) { + return void cb('E_BLOCK_DESERIALIZATION'); + } + BlockStore.write(Env, publicKey, buffer, function (err) { Env.Log.info('BLOCK_WRITE_BY_OWNER', { safeKey: safeKey, blockId: publicKey, @@ -185,7 +191,7 @@ Block.removeLoginBlock = function (Env, safeKey, msg, _cb) { return void cb("INCORRECT_KEY"); } }).nThen(function () { - validateLoginBlock(Env, publicKey, signature, DELETE_BLOCK, function (e /*::, validatedBlock */) { + Env.validateLoginBlock(publicKey, signature, DELETE_BLOCK, function (e) { if (e) { return void cb(e); } BlockStore.archive(Env, publicKey, function (err) { Env.Log.info('ARCHIVAL_BLOCK_BY_OWNER_RPC', { diff --git a/lib/workers/db-worker.js b/lib/workers/db-worker.js index 137b756b4..2371240ec 100644 --- a/lib/workers/db-worker.js +++ b/lib/workers/db-worker.js @@ -697,6 +697,10 @@ COMMANDS.VALIDATE_ANCESTOR_PROOF = function (data, cb) { Block.validateAncestorProof(Env, data && data.proof, cb); }; +COMMANDS.VALIDATE_LOGIN_BLOCK = function (data, cb) { + Block.validateLoginBlock(Env, data.publicKey, data.signature, data.block, cb); +}; + process.on('message', function (data) { if (!data || !data.txid || !data.pid) { return void process.send({ diff --git a/lib/workers/index.js b/lib/workers/index.js index 85c66eeb5..c2bfb5740 100644 --- a/lib/workers/index.js +++ b/lib/workers/index.js @@ -451,6 +451,15 @@ Workers.initialize = function (Env, config, _cb) { }, cb); }; + Env.validateLoginBlock = function (publicKey, signature, block, cb) { + sendCommand({ + command: 'VALIDATE_LOGIN_BLOCK', + publicKey: publicKey, + signature: signature, + block: block, + }, cb); + }; + cb(void 0); }); }; From fe4b85c98b01d829c0d00d56e9fddffe74768c62 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Jun 2021 15:55:59 +0530 Subject: [PATCH 30/79] remove hardcoded translations for blocked images --- www/common/diffMarked.js | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index 1f4fadefb..f4c5f40cd 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -279,12 +279,8 @@ define([ } }; - Messages.resources_imageBlocked = "CryptPad blocked a remote image"; // XXX - Messages.resources_openInNewTab = "Open its source in a new tab"; // XXX - Messages.resources_learnWhy = "Learn why it was blocked"; // XXX - renderer.image = function (href, title, text) { - if (href.slice(0,6) === '/file/') { // XXX this has been deprecated for about 3 years. Maybe we should display a warning? + if (href.slice(0,6) === '/file/') { // FIXME this has been deprecated for about 3 years. Maybe we should display a warning? // DEPRECATED // Mediatag using markdown syntax should not be used anymore so they don't support // password-protected files @@ -305,7 +301,7 @@ define([ h('div.cp-inline-img', [ h('img.cp-inline-img', { src: '/images/broken.png', - //title: title || '', // XXX sort out tippy issues (double-title) + //title: title || '', // FIXME sort out tippy issues (double-title) }), h('p.cp-alt-txt', text), ]), @@ -315,14 +311,12 @@ define([ h('a.cp-remote-img', { href: qualifiedHref(href), }, [ - //h('i.fa.fa-external-link'), // XXX Messages.resources_openInNewTab ]), h('br'), h('a.cp-learn-more', { - href: 'https://docs.cryptpad.fr/en/user_guide/index.html?placeholder=remote_images', // XXX point to an actual page + href: 'https://docs.cryptpad.fr/user_guide/security.html#remote-content', // XXX make sure this exists }, [ - //h('i.fa.fa-question-circle'), // XXX Messages.resources_learnWhy ]), ]); From 4fc60b86c5162af1975734c6cfb7b07d68668b27 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 29 Jun 2021 16:29:49 +0530 Subject: [PATCH 31/79] remove hardcoded admin panel translations --- www/admin/inner.js | 37 ++++++++----------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/www/admin/inner.js b/www/admin/inner.js index 592c77a5f..13f96cdf7 100644 --- a/www/admin/inner.js +++ b/www/admin/inner.js @@ -1854,24 +1854,6 @@ define([ }, }); - Messages.admin_instancePurposeTitle = "Instance purpose"; // XXX - Messages.admin_instancePurposeHint = "Why do you run this instance? Your answer will be shared with the developers unless you have disabled telemetry"; // XXX - - Messages.admin_purpose_noanswer = "I prefer not to answer"; // XXX - Messages.admin_purpose_experiment = "To test the CryptPad platform"; // XXX - Messages.admin_purpose_development = "To develop new features for CryptPad"; // XXX - - Messages.admin_purpose_personal = "For myself, family, or friends"; // XXX - - Messages.admin_purpose_business = "For my business's external use"; // XXX - Messages.admin_purpose_intranet = "For my business's internal use"; // XXX - - Messages.admin_purpose_school = "For my school, college, or university"; // XXX - Messages.admin_purpose_org = "For a non-profit organization or advocacy group"; // XXX - - Messages.admin_purpose_commercial = "To provide a commercial service"; // XXX - Messages.admin_purpose_public = "To provide a free service"; // XXX - var sendDecree = function (data, cb) { sFrameChan.query('Q_ADMIN_RPC', { cmd: 'ADMIN_DECREE', @@ -1881,19 +1863,16 @@ define([ create['instance-purpose'] = function () { var key = 'instance-purpose'; - var $div = makeBlock(key); + var $div = makeBlock(key); // Messages.admin_instancePurposeTitle.admin_instancePurposeHint var values = [ - 'noanswer', - 'development', - 'experiment', - 'personal', - 'business', // as a public resource for my business clients - 'intranet', // for my business's _internal use_ - 'school', - 'org', - 'commercial', - 'public', // to provide a free service (for the public) + 'noanswer', // Messages.admin_purpose_noanswer + 'experiment', // Messages.admin_purpose_experiment + 'personal', // Messages.admin_purpose_personal + 'education', // Messages.admin_purpose_education + 'org', // Messages.admin_purpose_org + 'business', // Messages.admin_purpose_business + 'public', // Messages.admin_purpose_public ]; var defaultPurpose = 'noanswer'; From 12c18fe7a3d6aa347df2749b5e26887d810f716c Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 30 Jun 2021 12:36:16 +0200 Subject: [PATCH 32/79] Remove unused file --- www/convert/file-crypto.js | 209 ------------------------------------- 1 file changed, 209 deletions(-) delete mode 100644 www/convert/file-crypto.js diff --git a/www/convert/file-crypto.js b/www/convert/file-crypto.js deleted file mode 100644 index 6a0c08816..000000000 --- a/www/convert/file-crypto.js +++ /dev/null @@ -1,209 +0,0 @@ -define([ - '/bower_components/tweetnacl/nacl-fast.min.js', -], function () { - var Nacl = window.nacl; - //var PARANOIA = true; - - var plainChunkLength = 128 * 1024; - var cypherChunkLength = 131088; - - var computeEncryptedSize = function (bytes, meta) { - var metasize = Nacl.util.decodeUTF8(JSON.stringify(meta)).length; - var chunks = Math.ceil(bytes / plainChunkLength); - return metasize + 18 + (chunks * 16) + bytes; - }; - - var encodePrefix = function (p) { - return [ - 65280, // 255 << 8 - 255, - ].map(function (n, i) { - return (p & n) >> ((1 - i) * 8); - }); - }; - var decodePrefix = function (A) { - return (A[0] << 8) | A[1]; - }; - - var slice = function (A) { - return Array.prototype.slice.call(A); - }; - - var createNonce = function () { - return new Uint8Array(new Array(24).fill(0)); - }; - - var increment = function (N) { - var l = N.length; - while (l-- > 1) { - /* our linter suspects this is unsafe because we lack types - but as long as this is only used on nonces, it should be safe */ - if (N[l] !== 255) { return void N[l]++; } // jshint ignore:line - if (l === 0) { throw new Error('E_NONCE_TOO_LARGE'); } - N[l] = 0; - } - }; - - var joinChunks = function (chunks) { - return new Blob(chunks); - }; - - var decrypt = function (u8, key, done, progress) { - var MAX = u8.length; - var _progress = function (offset) { - if (typeof(progress) !== 'function') { return; } - progress(Math.min(1, offset / MAX)); - }; - - var nonce = createNonce(); - var i = 0; - - var prefix = u8.subarray(0, 2); - var metadataLength = decodePrefix(prefix); - - var res = { - metadata: undefined, - }; - - var cancelled = false; - var cancel = function () { - cancelled = true; - }; - - var metaBox = new Uint8Array(u8.subarray(2, 2 + metadataLength)); - - var metaChunk = Nacl.secretbox.open(metaBox, nonce, key); - increment(nonce); - - try { - res.metadata = JSON.parse(Nacl.util.encodeUTF8(metaChunk)); - } catch (e) { - return window.setTimeout(function () { - done('E_METADATA_DECRYPTION'); - }); - } - - if (!res.metadata) { - return void setTimeout(function () { - done('NO_METADATA'); - }); - } - - var takeChunk = function (cb) { - setTimeout(function () { - var start = i * cypherChunkLength + 2 + metadataLength; - var end = start + cypherChunkLength; - i++; - var box = new Uint8Array(u8.subarray(start, end)); - - // decrypt the chunk - var plaintext = Nacl.secretbox.open(box, nonce, key); - increment(nonce); - - if (!plaintext) { return cb('DECRYPTION_ERROR'); } - - _progress(end); - cb(void 0, plaintext); - }); - }; - - var chunks = []; - - var again = function () { - if (cancelled) { return; } - takeChunk(function (e, plaintext) { - if (e) { - return setTimeout(function () { - done(e); - }); - } - if (plaintext) { - if ((2 + metadataLength + i * cypherChunkLength) < u8.length) { // not done - chunks.push(plaintext); - return setTimeout(again); - } - chunks.push(plaintext); - res.content = joinChunks(chunks); - return done(void 0, res); - } - done('UNEXPECTED_ENDING'); - }); - }; - - again(); - - return { - cancel: cancel - }; - }; - - // metadata - /* { filename: 'raccoon.jpg', type: 'image/jpeg' } */ - var encrypt = function (u8, metadata, key) { - var nonce = createNonce(); - - // encode metadata - var plaintext = Nacl.util.decodeUTF8(JSON.stringify(metadata)); - - // if metadata is too large, drop the thumbnail. - if (plaintext.length > 65535) { - var temp = JSON.parse(JSON.stringify(metadata)); - delete metadata.thumbnail; - plaintext = Nacl.util.decodeUTF8(JSON.stringify(temp)); - } - - var i = 0; - - var state = 0; - var next = function (cb) { - if (state === 2) { return void setTimeout(cb); } - - var start; - var end; - var part; - var box; - - if (state === 0) { // metadata... - part = new Uint8Array(plaintext); - box = Nacl.secretbox(part, nonce, key); - increment(nonce); - - if (box.length > 65535) { - return void cb('METADATA_TOO_LARGE'); - } - var prefixed = new Uint8Array(encodePrefix(box.length) - .concat(slice(box))); - state++; - - return void setTimeout(function () { - cb(void 0, prefixed); - }); - } - - // encrypt the rest of the file... - start = i * plainChunkLength; - end = start + plainChunkLength; - - part = u8.subarray(start, end); - box = Nacl.secretbox(part, nonce, key); - increment(nonce); - i++; - - // regular data is done - if (i * plainChunkLength >= u8.length) { state = 2; } - - setTimeout(function () { - cb(void 0, box); - }); - }; - - return next; - }; - - return { - decrypt: decrypt, - encrypt: encrypt, - joinChunks: joinChunks, - computeEncryptedSize: computeEncryptedSize, - }; -}); From 87d04ed68a446eee210ae1e73d864a76db8b5487 Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 30 Jun 2021 12:36:24 +0200 Subject: [PATCH 33/79] Fix form CSV export --- www/form/export.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/www/form/export.js b/www/form/export.js index e29c6dd87..93f3ff6d3 100644 --- a/www/form/export.js +++ b/www/form/export.js @@ -5,7 +5,7 @@ define([ var Export = {}; var escapeCSV = function (v) { - if (!/("|,)/.test(v)) { + if (!/("|,|\n)/.test(v)) { return v || ''; } var value = ''; @@ -14,7 +14,6 @@ define([ return value; }; Export.results = function (content, answers, TYPES) { - console.log(content, answers, TYPES); if (!content || !content.form) { return; } var csv = ""; var form = content.form; @@ -24,8 +23,8 @@ define([ if (!obj) { return; } return obj.q || Messages.form_default; }).filter(Boolean); - questions.unshift(Messages.form_poll_time); // "Time" questions.unshift(Messages.share_formView); // "Participant" + questions.unshift(Messages.form_poll_time); // "Time" questions.forEach(function (v, i) { if (i) { csv += ','; } @@ -37,14 +36,14 @@ define([ csv += '\n'; var time = new Date(obj.time).toISOString(); var msg = obj.msg || {}; - var user = msg._userdata; + var user = msg._userdata || {}; csv += escapeCSV(time); csv += ',' + escapeCSV(user.name || Messages.anonymous); Object.keys(form).forEach(function (key) { - csv += ',' + escapeCSV(String(msg[key])); + if (msg[key] && typeof(msg[key]) !== "string") { console.warn(key, msg[key]); } + csv += ',' + escapeCSV(String(msg[key] || '')); }); }); - console.log(csv); return csv; }; From c7ac75d137b7a508a6cd4fb7ca43fdbfdc2b83c7 Mon Sep 17 00:00:00 2001 From: yflory Date: Wed, 30 Jun 2021 12:47:02 +0200 Subject: [PATCH 34/79] Fix CSV export in forms --- www/form/export.js | 8 ++++++-- www/form/inner.js | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/www/form/export.js b/www/form/export.js index 93f3ff6d3..0e78d171a 100644 --- a/www/form/export.js +++ b/www/form/export.js @@ -5,7 +5,7 @@ define([ var Export = {}; var escapeCSV = function (v) { - if (!/("|,|\n)/.test(v)) { + if (!/("|,|\n|;)/.test(v)) { return v || ''; } var value = ''; @@ -40,7 +40,11 @@ define([ csv += escapeCSV(time); csv += ',' + escapeCSV(user.name || Messages.anonymous); Object.keys(form).forEach(function (key) { - if (msg[key] && typeof(msg[key]) !== "string") { console.warn(key, msg[key]); } + var type = form[key].type; + if (TYPES[type] && TYPES[type].exportCSV) { + csv += ',' + escapeCSV(TYPES[type].exportCSV(msg[key])); + return; + } csv += ',' + escapeCSV(String(msg[key] || '')); }); }); diff --git a/www/form/inner.js b/www/form/inner.js index 123e86d5b..8b51e2102 100644 --- a/www/form/inner.js +++ b/www/form/inner.js @@ -1646,6 +1646,15 @@ define([ return h('div.cp-form-type-poll', lines); }, + exportCSV: function (answer) { + if (!answer || !answer.values) { return ''; } + var str = ''; + Object.keys(answer.values).sort().forEach(function (k, i) { + if (i !== 0) { str += ';'; } + str += k.replace(';', '').replace(':', '') + ':' + answer.values[k]; + }); + return str; + }, icon: h('i.cptools.cptools-form-poll') }, }; From c9654a789267c01bf4c687c47d1978525cf84996 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 30 Jun 2021 16:51:50 +0530 Subject: [PATCH 35/79] remove XXX notes from forms and comment why they were there --- www/form/inner.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/www/form/inner.js b/www/form/inner.js index 123e86d5b..23332b944 100644 --- a/www/form/inner.js +++ b/www/form/inner.js @@ -80,8 +80,11 @@ define([ timeFormat = "h:i K"; } - var MAX_OPTIONS = 15; // XXX - var MAX_ITEMS = 10; // XXX + // multi-line radio, checkboxes, and possibly other things have a max number of items + // we'll consider increasing this restriction if people are unhappy with it + // but as a general rule we expect users will appreciate having simpler questions + var MAX_OPTIONS = 15; + var MAX_ITEMS = 10; var saveAndCancelOptions = function (getRes, cb) { // Cancel changes From 8e725f3d7c46f6720a78c081097cecb1181277a0 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 30 Jun 2021 17:20:03 +0530 Subject: [PATCH 36/79] stop returning the hash of all user pins after pinning the client doesn't use it and it's CPU-intensive --- lib/commands/pin-rpc.js | 12 +++++------- www/common/outer/async-store.js | 12 ++++++------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/lib/commands/pin-rpc.js b/lib/commands/pin-rpc.js index ff6bfdbac..af2c32c64 100644 --- a/lib/commands/pin-rpc.js +++ b/lib/commands/pin-rpc.js @@ -8,7 +8,7 @@ const nThen = require("nthen"); //const escapeKeyCharacters = Util.escapeKeyCharacters; const unescapeKeyCharacters = Util.unescapeKeyCharacters; -var sumChannelSizes = function (sizes) { +var sumChannelSizes = function (sizes) { // FIXME this synchronous code could be done by a worker return Object.keys(sizes).map(function (id) { return sizes[id]; }) .filter(function (x) { // only allow positive numbers @@ -171,7 +171,7 @@ Pinning.pinChannel = function (Env, safeKey, channels, cb) { getMultipleFileSize(Env, toStore, function (e, sizes) { if (typeof(sizes) === 'undefined') { return void cb(e); } - var pinSize = sumChannelSizes(sizes); + var pinSize = sumChannelSizes(sizes); // FIXME don't do this in the main thread... getFreeSpace(Env, safeKey, function (e, free) { if (typeof(free) === 'undefined') { @@ -186,7 +186,7 @@ Pinning.pinChannel = function (Env, safeKey, channels, cb) { toStore.forEach(function (channel) { session.channels[channel] = true; }); - getHash(Env, safeKey, cb); + cb(); }); }); }); @@ -217,7 +217,7 @@ Pinning.unpinChannel = function (Env, safeKey, channels, cb) { toStore.forEach(function (channel) { delete session.channels[channel]; }); - getHash(Env, safeKey, cb); + cb(); }); }); }; @@ -270,9 +270,7 @@ Pinning.resetUserPins = function (Env, safeKey, channelList, cb) { // update in-memory cache IFF the reset was allowed. session.channels = pins; - getHash(Env, safeKey, function (e, hash) { - cb(e, hash); - }); + cb(); }); }); }); diff --git a/www/common/outer/async-store.js b/www/common/outer/async-store.js index 66e1967cd..9f779a0a8 100644 --- a/www/common/outer/async-store.js +++ b/www/common/outer/async-store.js @@ -274,9 +274,9 @@ define([ } var pads = data.pads || data; - s.rpc.pin(pads, function (e, hash) { + s.rpc.pin(pads, function (e) { if (e) { return void cb({error: e}); } - cb({hash: hash}); + cb({}); }); }; @@ -289,9 +289,9 @@ define([ if (!s.rpc) { return void cb({error: 'RPC_NOT_READY'}); } var pads = data.pads || data; - s.rpc.unpin(pads, function (e, hash) { + s.rpc.unpin(pads, function (e) { if (e) { return void cb({error: e}); } - cb({hash: hash}); + cb({}); }); }; @@ -394,9 +394,9 @@ define([ if (!store.rpc) { return void cb({error: 'RPC_NOT_READY'}); } var list = getCanonicalChannelList(false); - store.rpc.reset(list, function (e, hash) { + store.rpc.reset(list, function (e) { if (e) { return void cb(e); } - cb(null, hash); + cb(null); }); }; From 76b90d3c8a8220aee0afd829d0eeaea081d248a6 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 30 Jun 2021 17:55:01 +0530 Subject: [PATCH 37/79] correct a few more places where the client expected hashes in pin responses or where the server incorrectly provided them --- lib/commands/pin-rpc.js | 17 ++++++++--------- scripts/tests/test-rpc.js | 22 ++++++++++------------ www/common/pinpad.js | 35 ++++++++++------------------------- 3 files changed, 28 insertions(+), 46 deletions(-) diff --git a/lib/commands/pin-rpc.js b/lib/commands/pin-rpc.js index af2c32c64..6fc05c202 100644 --- a/lib/commands/pin-rpc.js +++ b/lib/commands/pin-rpc.js @@ -145,7 +145,7 @@ var getFreeSpace = Pinning.getFreeSpace = function (Env, safeKey, cb) { }); }; -var getHash = Pinning.getHash = function (Env, safeKey, cb) { +Pinning.getHash = function (Env, safeKey, cb) { getChannelList(Env, safeKey, function (channels) { Env.hashChannelList(channels, cb); }); @@ -166,7 +166,7 @@ Pinning.pinChannel = function (Env, safeKey, channels, cb) { }); if (toStore.length === 0) { - return void getHash(Env, safeKey, cb); + return void cb(); } getMultipleFileSize(Env, toStore, function (e, sizes) { @@ -208,7 +208,7 @@ Pinning.unpinChannel = function (Env, safeKey, channels, cb) { }); if (toStore.length === 0) { - return void getHash(Env, safeKey, cb); + return void cb(); } Env.pinStore.message(safeKey, JSON.stringify(['UNPIN', toStore, +new Date()]), @@ -222,15 +222,14 @@ Pinning.unpinChannel = function (Env, safeKey, channels, cb) { }); }; -Pinning.resetUserPins = function (Env, safeKey, channelList, cb) { +Pinning.resetUserPins = function (Env, safeKey, channelList, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); if (!Array.isArray(channelList)) { return void cb('INVALID_PIN_LIST'); } var session = Core.getSession(Env.Sessions, safeKey); - if (!channelList.length) { - return void getHash(Env, safeKey, function (e, hash) { - if (e) { return cb(e); } - cb(void 0, hash); - }); + + if (!channelList.length) { // XXX wut + return void cb(); } var pins = {}; diff --git a/scripts/tests/test-rpc.js b/scripts/tests/test-rpc.js index 0fd0abc37..b92cf6287 100644 --- a/scripts/tests/test-rpc.js +++ b/scripts/tests/test-rpc.js @@ -122,14 +122,11 @@ var createUser = function (config, cb) { }); })); }).nThen(function (w) { - user.rpc.reset([], w(function (err, hash) { + user.rpc.reset([], w(function (err) { if (err) { w.abort(); user.shutdown(); - return console.log("RESET_ERR"); - } - if (!hash || hash !== EMPTY_ARRAY_HASH) { - throw new Error("EXPECTED EMPTY ARRAY HASH"); + return console.log("TEST_RESET_ERR"); } })); }).nThen(function (w) { @@ -214,17 +211,17 @@ var createUser = function (config, cb) { // TODO check your quota usage }).nThen(function (w) { - user.rpc.unpin([user.mailboxChannel], w(function (err, hash) { + user.rpc.unpin([user.mailboxChannel], w(function (err) { if (err) { w.abort(); return void cb(err); } + })); + }).nThen(function (w) { + user.rpc.getServerHash(w(function (err, hash) { + console.log(hash); - if (hash[0] !== EMPTY_ARRAY_HASH) { - //console.log('UNPIN_RESPONSE', hash); - throw new Error("UNPIN_DIDNT_WORK"); - } - user.latestPinHash = hash[0]; + user.latestPinHash = hash; })); }).nThen(function (w) { // clean up the pin list to avoid lots of accounts on the server @@ -304,7 +301,8 @@ nThen(function (w) { }, w(function (err, roster) { if (err) { w.abort(); - return void console.trace(err); + console.error(err); + return void console.error("ROSTER_ERROR"); } oscar.roster = roster; oscar.destroy.reg(function () { diff --git a/www/common/pinpad.js b/www/common/pinpad.js index 7e9cd4ee2..de93f066f 100644 --- a/www/common/pinpad.js +++ b/www/common/pinpad.js @@ -26,23 +26,19 @@ var factory = function (Util, Rpc) { exp.send = rpc.send; // you can ask the server to pin a particular channel for you - exp.pin = function (channels, cb) { + exp.pin = function (channels, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); if (!Array.isArray(channels)) { - setTimeout(function () { - cb('[TypeError] pin expects an array'); - }); - return; + return void cb('[TypeError] pin expects an array'); } rpc.send('PIN', channels, cb); }; // you can also ask to unpin a particular channel - exp.unpin = function (channels, cb) { + exp.unpin = function (channels, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); if (!Array.isArray(channels)) { - setTimeout(function () { - cb('[TypeError] pin expects an array'); - }); - return; + return void cb('[TypeError] pin expects an array'); } rpc.send('UNPIN', channels, cb); }; @@ -70,23 +66,12 @@ var factory = function (Util, Rpc) { }; // if local and remote hashes don't match, send a reset - exp.reset = function (channels, cb) { + exp.reset = function (channels, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); if (!Array.isArray(channels)) { - setTimeout(function () { - cb('[TypeError] pin expects an array'); - }); - return; + return void cb('[TypeError] pin expects an array'); } - rpc.send('RESET', channels, function (e, response) { - if (e) { - return void cb(e); - } - if (!response.length) { - console.log(response); - return void cb('INVALID_RESPONSE'); - } - cb(e, response[0]); - }); + rpc.send('RESET', channels, cb); }; // get the combined size of all channels (in bytes) for all the From e57ccf14d70dcbabd15e8d4944c1c6787765e747 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 30 Jun 2021 18:20:57 +0530 Subject: [PATCH 38/79] clean up some pending notes --- www/admin/inner.js | 3 +-- www/convert/inner.js | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/www/admin/inner.js b/www/admin/inner.js index 13f96cdf7..263f9a9bc 100644 --- a/www/admin/inner.js +++ b/www/admin/inner.js @@ -94,7 +94,7 @@ define([ 'cp-admin-list-my-instance', 'cp-admin-consent-to-contact', 'cp-admin-remove-donate-button', - 'cp-admin-instance-purpose', // XXX + 'cp-admin-instance-purpose', ], }; @@ -1902,7 +1902,6 @@ define([ [ value] ], cb); }; - //var spinner = UI.makeSpinner($br); // XXX $opts.on('change', function () { var val = $opts.find('input:radio:checked').val(); diff --git a/www/convert/inner.js b/www/convert/inner.js index d8eca9af2..4952b2d6e 100644 --- a/www/convert/inner.js +++ b/www/convert/inner.js @@ -194,7 +194,7 @@ define([ }; Messages.convertPage = "Convert"; // XXX - Messages.convert_hint = "Pick the file you want to convert. The list of output format will be visible afterward."; + Messages.convert_hint = "Pick the file you want to convert. The list of output format will be visible afterward."; // XXX var createToolbar = function () { var displayed = ['useradmin', 'newpad', 'limit', 'pageTitle', 'notifications']; From f2f9ace7c63ae7403a7e79c18a14f65828fd7c72 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 30 Jun 2021 18:53:32 +0530 Subject: [PATCH 39/79] fix rendering issue with markdown media-tag format --- www/common/diffMarked.js | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index 1919f0670..d7c476a90 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -287,9 +287,28 @@ define([ } }; + var isLocalURL = function (href) { + // treat all URLs as remote if you are using an ancient browser + if (typeof(window.URL) === 'undefined') { return false; } + try { + var url = new URL(href, ApiConfig.httpUnsafeOrigin); + var localURL = new URL(ApiConfig.httpUnsafeOrigin); + return url.host === localURL.host; + } catch (err) { + return true; + } + }; + renderer.image = function (href, title, text) { - if (href.slice(0,6) === '/file/') { // FIXME this has been deprecated for about 3 years. Maybe we should display a warning? - // DEPRECATED + if (isLocalURL(href) && href.slice(0, 6) !== '/file/') { + return h('img', { + src: href, + title: title || '', + alt: text, + }).outerHTML; + } + + if (href.slice(0,6) === '/file/') { // Mediatag using markdown syntax should not be used anymore so they don't support // password-protected files console.log('DEPRECATED: mediatag using markdown syntax!'); @@ -297,12 +316,11 @@ define([ var secret = Hash.getSecrets('file', parsed.hash); var src = (ApiConfig.fileHost || '') +Hash.getBlobPathFromHex(secret.channel); var key = Hash.encodeBase64(secret.keys.cryptKey); - var mt = ''; - if (mediaMap[src]) { - mt += mediaMap[src]; - } - mt += ''; - return mt; + var mt = h('media-tag', { + src: src, + 'data-crypto-key': 'cryptpad:' + key, + }); + return mt.outerHTML; } var warning = h('div.cp-inline-img-warning', [ @@ -323,7 +341,7 @@ define([ ]), h('br'), h('a.cp-learn-more', { - href: 'https://docs.cryptpad.fr/user_guide/security.html#remote-content', // XXX make sure this exists + href: 'https://docs.cryptpad.fr/user_guide/security.html#remote-content', }, [ Messages.resources_learnWhy ]), From d27cbb69dc3686dd17680229d61f198bde7edefd Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 30 Jun 2021 19:05:37 +0530 Subject: [PATCH 40/79] filter email and instance purpose from telemetry unless we have consent --- lib/stats.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/stats.js b/lib/stats.js index 2a513da44..1dbbb2ad9 100644 --- a/lib/stats.js +++ b/lib/stats.js @@ -12,10 +12,10 @@ Stats.instanceData = function (Env) { httpUnsafeOrigin: Env.httpUnsafeOrigin, httpSafeOrigin: Env.httpSafeOrigin, - adminEmail: Env.adminEmail, + adminEmail: Env.consentToContact? Env.adminEmail: undefined, consentToContact: Boolean(Env.consentToContact), - instancePurpose: Env.instancePurpose, // XXX + instancePurpose: Env.instancePurpose === 'noanswer'? undefined: Env.instancePurpose, }; /* We reserve the right to choose not to include instances From f7f2146fa53e94be742f607a4a089e1c2874f1e0 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 30 Jun 2021 19:09:53 +0530 Subject: [PATCH 41/79] miscellaneous cleanup and notes --- customize.dist/src/less2/include/markdown.less | 4 ++-- lib/commands/pin-rpc.js | 2 +- lib/env.js | 2 +- lib/storage/block.js | 11 ----------- 4 files changed, 4 insertions(+), 15 deletions(-) diff --git a/customize.dist/src/less2/include/markdown.less b/customize.dist/src/less2/include/markdown.less index f52b2666f..f42a27146 100644 --- a/customize.dist/src/less2/include/markdown.less +++ b/customize.dist/src/less2/include/markdown.less @@ -177,12 +177,12 @@ a { color: @cryptpad_text_col; font-size: 0.8em; - &.cp-remote-img::before { + &.cp-remote-img::before { // .fa.fa-question-circle font-family: FontAwesome; //content: "\f08e\00a0"; content: "\f08e\00a0\00a0"; } - &.cp-learn-more::before { + &.cp-learn-more::before { // .fa.fa-external-link font-family: FontAwesome; content: "\f059\00a0"; //content: "\f059\00a0\00a0"; diff --git a/lib/commands/pin-rpc.js b/lib/commands/pin-rpc.js index 6fc05c202..89afc4ee8 100644 --- a/lib/commands/pin-rpc.js +++ b/lib/commands/pin-rpc.js @@ -228,7 +228,7 @@ Pinning.resetUserPins = function (Env, safeKey, channelList, _cb) { var session = Core.getSession(Env.Sessions, safeKey); - if (!channelList.length) { // XXX wut + if (!channelList.length) { return void cb(); } diff --git a/lib/env.js b/lib/env.js index 6b033fa16..141e028ae 100644 --- a/lib/env.js +++ b/lib/env.js @@ -123,7 +123,7 @@ module.exports.create = function (config) { maxWorkers: config.maxWorkers, disableIntegratedTasks: config.disableIntegratedTasks || false, - disableIntegratedEviction: config.disableIntegratedEviction || false, + disableIntegratedEviction: config.disableIntegratedEviction || true, // XXX false, lastEviction: +new Date(), evictionReport: {}, commandTimers: {}, diff --git a/lib/storage/block.js b/lib/storage/block.js index 5dfc393a5..d3e7a1869 100644 --- a/lib/storage/block.js +++ b/lib/storage/block.js @@ -79,14 +79,3 @@ Block.write = function (Env, publicKey, buffer, _cb) { }); }; -/* -Block.create = function (opt, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - - var env = { - root: opt.root || '', // XXX - - - }; -}; -*/ From 0978074c74e479a34b972cfdc01bfd2f6597a254 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 30 Jun 2021 19:31:48 +0530 Subject: [PATCH 42/79] add convert app to example nginx and update changelog --- CHANGELOG.md | 20 ++++++++++++++++++++ docs/example.nginx.conf | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b41b1d9b0..18fd928a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ * server * `installMethod: 'unspecified'` in the default config to distinguish docker installs * `instancePurpose` on admin panel + * add support for archiving pin lists (instead of deleting them) + * blocks + * archive blocks instead of deleting them outright + * implement as storage API + * validate blocks in worker + * don't include adminEmail in telemetry unless we have consentToContact + * don't include instancePurpose in telemetry if it is "noanswer" * display warnings when remote resources are blocked * in code preview * restrict style tags to a scope when rendering them in markdown preview by compiling their content as scoped less @@ -15,6 +22,19 @@ * form templates * guard against a type error in `getAccessKeys` * guard against invalid or malicious input when constructing media-tags for embedding in markdown +* Japanese translation +* conversions + * convert app + * some basic office formats + * rich text => markdown + * handle some pecularities with headings and ids + * forms => .csv + * trello import with some loss +* fix rendering issue for legacy markdown media tag syntax + * guard against domExceptions + * catch errors thrown by the diff applier +* don't bother returning the hash of a pin list to the client, since they don't use it + # 4.7.0 diff --git a/docs/example.nginx.conf b/docs/example.nginx.conf index 85f42dd81..29317ee27 100644 --- a/docs/example.nginx.conf +++ b/docs/example.nginx.conf @@ -214,7 +214,7 @@ server { # The nodejs server has some built-in forwarding rules to prevent # URLs like /pad from resulting in a 404. This simply adds a trailing slash # to a variety of applications. - location ~ ^/(register|login|settings|user|pad|drive|poll|slide|code|whiteboard|file|media|profile|contacts|todo|filepicker|debug|kanban|sheet|support|admin|notifications|teams|calendar|presentation|doc|form|report)$ { + location ~ ^/(register|login|settings|user|pad|drive|poll|slide|code|whiteboard|file|media|profile|contacts|todo|filepicker|debug|kanban|sheet|support|admin|notifications|teams|calendar|presentation|doc|form|report|convert)$ { rewrite ^(.*)$ $1/ redirect; } From 575269e5a1fbcc7d5b96e47f7ca50ad77905326e Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 1 Jul 2021 13:08:35 +0530 Subject: [PATCH 43/79] use a standard function for creating basic media-tags --- www/code/inner.js | 2 +- www/common/common-interface.js | 9 +++++++++ www/common/diffMarked.js | 1 - www/common/inner/common-mediatag.js | 5 ++--- www/common/sframe-app-framework.js | 4 ++-- www/common/sframe-common.js | 3 +-- www/kanban/inner.js | 2 +- www/poll/inner.js | 4 ++-- www/slide/inner.js | 2 +- 9 files changed, 19 insertions(+), 13 deletions(-) diff --git a/www/code/inner.js b/www/code/inner.js index d74513035..785e82788 100644 --- a/www/code/inner.js +++ b/www/code/inner.js @@ -507,7 +507,7 @@ define([ var fileHost = privateData.fileHost || privateData.origin; var src = fileHost + Hash.getBlobPathFromHex(secret.channel); var key = Hash.encodeBase64(secret.keys.cryptKey); - var mt = ''; + var mt = UI.mediaTag(src, key).outerHTML; editor.replaceSelection(mt); } }; diff --git a/www/common/common-interface.js b/www/common/common-interface.js index 9bdf9ec7a..4d3520426 100644 --- a/www/common/common-interface.js +++ b/www/common/common-interface.js @@ -41,6 +41,15 @@ define([ return e; }; + // FIXME almost everywhere this is used would also be + // a good candidate for sframe-common's getMediatagFromHref + UI.mediaTag = function (src, key) { + return h('media-tag', { + src: src, + 'data-crypto-key': 'cryptpad:' + key, + }); + }; + var findCancelButton = UI.findCancelButton = function (root) { if (root) { return $(root).find('button.cancel').last(); diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index d7c476a90..52a8cf0a1 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -676,7 +676,6 @@ define([ if (el.childNodes.length === 1 && el.childNodes[0].nodeType === 3) { var type = el.getAttribute('data-plugin'); var plugin = plugins[type]; - console.log(type); if (!plugin) { return; } var src = canonicalizeMermaidSource(el.childNodes[0].wholeText); el.setAttribute(plugin.attr, src); diff --git a/www/common/inner/common-mediatag.js b/www/common/inner/common-mediatag.js index 966724fa1..b8550fcac 100644 --- a/www/common/inner/common-mediatag.js +++ b/www/common/inner/common-mediatag.js @@ -127,9 +127,8 @@ define([ if (e || !data) { return void displayDefault(); } if (typeof data !== "number") { return void displayDefault(); } if (Util.bytesToMegabytes(data) > 0.5) { return void displayDefault(); } - var $img = $('').appendTo($container); - $img.attr('src', src); - $img.attr('data-crypto-key', 'cryptpad:' + cryptKey); + var mt = UI.mediaTag(src, cryptKey); + var $img = $(mt).appendTo($container); MT.displayMediatagImage(common, $img, function (err, $image) { if (err) { return void console.error(err); } centerImage($img, $image); diff --git a/www/common/sframe-app-framework.js b/www/common/sframe-app-framework.js index 796640f6d..0fd250f80 100644 --- a/www/common/sframe-app-framework.js +++ b/www/common/sframe-app-framework.js @@ -748,8 +748,8 @@ define([ var privateDat = cpNfInner.metadataMgr.getPrivateData(); var origin = privateDat.fileHost || privateDat.origin; var src = data.src = data.src.slice(0,1) === '/' ? origin + data.src : data.src; - mediaTagEmbedder($(''), data); + var mt = UI.mediaTag(src, data.key); + mediaTagEmbedder($(mt), data); }); }).appendTo(toolbar.$bottomL).hide(); }; diff --git a/www/common/sframe-common.js b/www/common/sframe-common.js index 3fd4869d1..5d3738cbb 100644 --- a/www/common/sframe-common.js +++ b/www/common/sframe-common.js @@ -145,8 +145,7 @@ define([ var hexFileName = secret.channel; var origin = data.fileHost || data.origin; var src = origin + Hash.getBlobPathFromHex(hexFileName); - return '' + - ''; + return UI.mediaTag(src, key).outerHTML; } return; }; diff --git a/www/kanban/inner.js b/www/kanban/inner.js index 8eb090ffc..4832adcca 100644 --- a/www/kanban/inner.js +++ b/www/kanban/inner.js @@ -289,7 +289,7 @@ define([ var fileHost = privateData.fileHost || privateData.origin; var src = fileHost + Hash.getBlobPathFromHex(secret.channel); var key = Hash.encodeBase64(secret.keys.cryptKey); - var mt = ''; + var mt = UI.mediaTag(src, key).outerHTML; editor.replaceSelection(mt); } }; diff --git a/www/poll/inner.js b/www/poll/inner.js index 6d4e0590e..53faea7e9 100644 --- a/www/poll/inner.js +++ b/www/poll/inner.js @@ -955,7 +955,7 @@ define([ var fileHost = privateData.fileHost || privateData.origin; var src = fileHost + Hash.getBlobPathFromHex(secret.channel); var key = Hash.encodeBase64(secret.keys.cryptKey); - var mt = ''; + var mt = UI.mediaTag(src, key).outerHTML; APP.editor.replaceSelection(mt); } }; @@ -1235,7 +1235,7 @@ define([ common.openFilePicker(pickerCfg, function (data) { if (data.type === 'file' && APP.editor) { common.setPadAttribute('atime', +new Date(), null, data.href); - var mt = ''; + var mt = UI.mediaTag(data.src, data.key).outerHTML; APP.editor.replaceSelection(mt); return; } diff --git a/www/slide/inner.js b/www/slide/inner.js index 3eb1435ed..e40bd6d79 100644 --- a/www/slide/inner.js +++ b/www/slide/inner.js @@ -556,7 +556,7 @@ define([ var fileHost = privateData.fileHost || privateData.origin; var src = fileHost + Hash.getBlobPathFromHex(secret.channel); var key = Hash.encodeBase64(secret.keys.cryptKey); - var mt = ''; + var mt = UI.mediaTag(src, key).outerHTML; editor.replaceSelection(mt); } }; From 7d634ab54b90a0e1efe28edb48dd13d17ee6038b Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 1 Jul 2021 13:58:17 +0530 Subject: [PATCH 44/79] suppress 'convert' from new pad modal and make it easy to configure what's displayed via a list in appconfig --- www/common/application_config_internal.js | 6 ++++++ www/common/common-ui-elements.js | 10 ++-------- www/common/drive-ui.js | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/www/common/application_config_internal.js b/www/common/application_config_internal.js index 68ba4ed06..971da32ee 100644 --- a/www/common/application_config_internal.js +++ b/www/common/application_config_internal.js @@ -22,6 +22,12 @@ define(function() { */ AppConfig.registeredOnlyTypes = ['file', 'contacts', 'notifications', 'support']; + // XXX to prevent apps that aren't officially supported from showing up + // in the document creation modal + AppConfig.hiddenTypes = ['drive', 'teams', 'contacts', 'todo', 'file', 'accounts', 'calendar', 'poll', 'convert', + //'doc', 'presentation' + ]; + /* CryptPad is available is multiple languages, but only English and French are maintained * by the developers. The other languages may be outdated, and any missing string for a langauge * will use the english version instead. You can customize the langauges you want to be available diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index 3e27b4ee7..a7ef39de6 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -2075,15 +2075,9 @@ define([ var $container = $('
    '); var i = 0; + var types = AppConfig.availablePadTypes.filter(function (p) { - if (p === 'drive') { return; } - if (p === 'teams') { return; } - if (p === 'contacts') { return; } - if (p === 'todo') { return; } - if (p === 'file') { return; } - if (p === 'accounts') { return; } - if (p === 'calendar') { return; } - if (p === 'poll') { return; } // Replaced by forms + if (AppConfig.hiddenTypes.indexOf(p) !== -1) { return; } if (!common.isLoggedIn() && AppConfig.registeredOnlyTypes && AppConfig.registeredOnlyTypes.indexOf(p) !== -1) { return; } return true; diff --git a/www/common/drive-ui.js b/www/common/drive-ui.js index 4a75459a5..0db7018eb 100644 --- a/www/common/drive-ui.js +++ b/www/common/drive-ui.js @@ -3082,6 +3082,7 @@ define([ } // Pads getNewPadTypes().forEach(function (type) { + if (AppConfig.hiddenTypes.indexOf(type) !== -1) { return; } var $element = $('
  • ', { 'class': 'cp-app-drive-new-doc cp-app-drive-element-row ' + 'cp-app-drive-element-grid' From 899eef1ee87d7ddb013efba8f849c36ba9a1b0a8 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 1 Jul 2021 14:01:43 +0530 Subject: [PATCH 45/79] filter hidden types from a more sensible place in the drive --- www/common/drive-ui.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/www/common/drive-ui.js b/www/common/drive-ui.js index 0db7018eb..e079ad2b9 100644 --- a/www/common/drive-ui.js +++ b/www/common/drive-ui.js @@ -2571,14 +2571,7 @@ define([ var getNewPadTypes = function () { var arr = []; AppConfig.availablePadTypes.forEach(function (type) { - if (type === 'drive') { return; } - if (type === 'teams') { return; } - if (type === 'contacts') { return; } - if (type === 'todo') { return; } - if (type === 'file') { return; } - if (type === 'accounts') { return; } - if (type === 'calendar') { return; } - if (type === 'poll') { return; } // replaced by forms + if (AppConfig.hiddenTypes.indexOf(type) !== -1) { return; } if (!APP.loggedIn && AppConfig.registeredOnlyTypes && AppConfig.registeredOnlyTypes.indexOf(type) !== -1) { return; @@ -3082,7 +3075,6 @@ define([ } // Pads getNewPadTypes().forEach(function (type) { - if (AppConfig.hiddenTypes.indexOf(type) !== -1) { return; } var $element = $('
  • ', { 'class': 'cp-app-drive-new-doc cp-app-drive-element-row ' + 'cp-app-drive-element-grid' From 3b44c09bc4b0bfe4c9b7933142fd0f5c660cfeaf Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 1 Jul 2021 16:42:09 +0530 Subject: [PATCH 46/79] check COOP headers for multiple endpoints and improve some error reporting in the checkup RPC --- docs/example.nginx.conf | 2 +- server.js | 2 +- www/checkup/main.js | 30 ++++++++++++++++++++++++++++++ www/checkup/sandbox/main.js | 12 +++++++++--- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/docs/example.nginx.conf b/docs/example.nginx.conf index 29317ee27..8bc47d9f8 100644 --- a/docs/example.nginx.conf +++ b/docs/example.nginx.conf @@ -64,7 +64,7 @@ server { add_header Permissions-Policy interest-cohort=(); set $coop ''; - if ($uri ~ ^\/(sheet|presentation|doc)\/.*$) { set $coop 'same-origin'; } + if ($uri ~ ^\/(sheet|presentation|doc|convert)\/.*$) { set $coop 'same-origin'; } # Enable SharedArrayBuffer in Firefox (for .xlsx export) add_header Cross-Origin-Resource-Policy cross-origin; diff --git a/server.js b/server.js index 848167a98..3cea171c6 100644 --- a/server.js +++ b/server.js @@ -90,7 +90,7 @@ var setHeaders = (function () { return function (req, res) { // apply a bunch of cross-origin headers for XLSX export in FF and printing elsewhere applyHeaderMap(res, { - "Cross-Origin-Opener-Policy": /^\/sheet\//.test(req.url)? 'same-origin': '', + "Cross-Origin-Opener-Policy": /^\/(sheet|presentation|doc|convert)\//.test(req.url)? 'same-origin': '', }); if (Env.NO_SANDBOX) { // handles correct configuration for local development diff --git a/www/checkup/main.js b/www/checkup/main.js index 732197a2d..143d1709b 100644 --- a/www/checkup/main.js +++ b/www/checkup/main.js @@ -732,6 +732,36 @@ define([ cb(isHTTPS(trimmedUnsafe) && isHTTPS(trimmedSafe)); }); + + [ + 'sheet', + 'presentation', + 'doc', + 'convert', + ].forEach(function (url) { + assert(function (cb, msg) { + var header = 'cross-origin-opener-policy'; + var expected = 'same-origin'; + deferredPostMessage({ + command: 'GET_HEADER', + content: { + url: '/' + url + '/', + header: header, + } + }, function (content) { + msg.appendChild(h('span', [ + code(url), + ' was served without the correct ', + code(header), + ' HTTP header value (', + code(expected), + '). This will interfere with your ability to convert between office file formats.' + ])); + cb(content === expected); + }); + }); + }); + /* assert(function (cb, msg) { setWarningClass(msg); diff --git a/www/checkup/sandbox/main.js b/www/checkup/sandbox/main.js index 7ddfb07f8..e11aa1d52 100644 --- a/www/checkup/sandbox/main.js +++ b/www/checkup/sandbox/main.js @@ -27,12 +27,14 @@ define([ }; window.addEventListener("message", function (event) { + var txid, command; if (event && event.data) { try { //console.log(JSON.parse(event.data)); var msg = JSON.parse(event.data); - var command = msg.command; - var txid = msg.txid; + command = msg.command; + txid = msg.txid; + if (!txid) { return; } COMMANDS[command](msg.content, function (response) { // postMessage with same txid postMessage({ @@ -41,7 +43,11 @@ define([ }); }); } catch (err) { - console.error(err); + postMessage({ + txid: txid, + content: err, + }); + console.error(err, command); } } else { console.error(event); From 1c1dc421a324fd6c5b60a0aee8af97e8761fdbdd Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 1 Jul 2021 16:44:40 +0530 Subject: [PATCH 47/79] fix broken block archival --- lib/commands/block.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/commands/block.js b/lib/commands/block.js index ea6a19dc2..195e0c494 100644 --- a/lib/commands/block.js +++ b/lib/commands/block.js @@ -167,7 +167,7 @@ Block.writeLoginBlock = function (Env, safeKey, msg, _cb) { }); }; -const DELETE_BLOCK = Nacl.util.decodeUTF8('DELETE_BLOCK'); +const DELETE_BLOCK = Nacl.util.encodeBase64(Nacl.util.decodeUTF8('DELETE_BLOCK')); /* When users write a block, they upload the block, and provide From 279db28e86e0c2cab14ab0e2a14153cc7774efa2 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 1 Jul 2021 18:44:35 +0530 Subject: [PATCH 48/79] start tracking the versions of our vendored software --- www/lib/changelog.md | 4 + www/lib/turndown.browser.umd.js | 135 ++++++++++++++++++-------------- 2 files changed, 80 insertions(+), 59 deletions(-) create mode 100644 www/lib/changelog.md diff --git a/www/lib/changelog.md b/www/lib/changelog.md new file mode 100644 index 000000000..86ef6ec22 --- /dev/null +++ b/www/lib/changelog.md @@ -0,0 +1,4 @@ +This file is intended to be used as a log of what third-party source we have vendored, where we got it, and what modifications we have made to it (if any). + +* [turndown v7.1.1](https://github.com/mixmark-io/turndown/releases/tag/v7.1.1) built from unmodified source as per its build scripts. + diff --git a/www/lib/turndown.browser.umd.js b/www/lib/turndown.browser.umd.js index 9b8c40d13..a812101ae 100644 --- a/www/lib/turndown.browser.umd.js +++ b/www/lib/turndown.browser.umd.js @@ -1,7 +1,7 @@ (function (global, factory) { typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() : typeof define === 'function' && define.amd ? define(factory) : - (global = global || self, global.TurndownService = factory()); + (global = typeof globalThis !== 'undefined' ? globalThis : global || self, global.TurndownService = factory()); }(this, (function () { 'use strict'; function extend (destination) { @@ -18,6 +18,17 @@ return Array(count + 1).join(character) } + function trimLeadingNewlines (string) { + return string.replace(/^\n*/, '') + } + + function trimTrailingNewlines (string) { + // avoid match-at-end regexp bottleneck, see #370 + var indexEnd = string.length; + while (indexEnd > 0 && string[indexEnd - 1] === '\n') indexEnd--; + return string.substring(0, indexEnd) + } + var blockElements = [ 'ADDRESS', 'ARTICLE', 'ASIDE', 'AUDIO', 'BLOCKQUOTE', 'BODY', 'CANVAS', 'CENTER', 'DD', 'DIR', 'DIV', 'DL', 'DT', 'FIELDSET', 'FIGCAPTION', 'FIGURE', @@ -303,19 +314,15 @@ }, replacement: function (content) { - if (!content.trim()) return '' + if (!content) return '' + content = content.replace(/\r?\n|\r/g, ' '); + var extraSpace = /^`|^ .*?[^ ].* $|`$/.test(content) ? ' ' : ''; var delimiter = '`'; - var leadingSpace = ''; - var trailingSpace = ''; - var matches = content.match(/`+/gm); - if (matches) { - if (/^`/.test(content)) leadingSpace = ' '; - if (/`$/.test(content)) trailingSpace = ' '; - while (matches.indexOf(delimiter) !== -1) delimiter = delimiter + '`'; - } + var matches = content.match(/`+/gm) || []; + while (matches.indexOf(delimiter) !== -1) delimiter = delimiter + '`'; - return delimiter + leadingSpace + content + trailingSpace + delimiter + return delimiter + extraSpace + content + extraSpace + delimiter } }; @@ -459,7 +466,7 @@ if (!element.firstChild || isPre(element)) return var prevText = null; - var prevVoid = false; + var keepLeadingWs = false; var prev = null; var node = next(prev, element, isPre); @@ -469,7 +476,7 @@ var text = node.data.replace(/[ \r\n\t]+/g, ' '); if ((!prevText || / $/.test(prevText.data)) && - !prevVoid && text[0] === ' ') { + !keepLeadingWs && text[0] === ' ') { text = text.substr(1); } @@ -489,11 +496,14 @@ } prevText = null; - prevVoid = false; - } else if (isVoid(node)) { - // Avoid trimming space around non-block, non-BR void elements. + keepLeadingWs = false; + } else if (isVoid(node) || isPre(node)) { + // Avoid trimming space around non-block, non-BR void elements and inline PRE. prevText = null; - prevVoid = true; + keepLeadingWs = true; + } else if (prevText) { + // Drop protection if set previously. + keepLeadingWs = false; } } else { node = remove(node); @@ -609,7 +619,7 @@ var HTMLParser = canParseHTMLNatively() ? root.DOMParser : createHTMLParser(); - function RootNode (input) { + function RootNode (input, options) { var root; if (typeof input === 'string') { var doc = htmlParser().parseFromString( @@ -626,7 +636,8 @@ collapseWhitespace({ element: root, isBlock: isBlock, - isVoid: isVoid + isVoid: isVoid, + isPre: options.preformattedCode ? isPreOrCode : null }); return root @@ -638,11 +649,15 @@ return _htmlParser } - function Node (node) { + function isPreOrCode (node) { + return node.nodeName === 'PRE' || node.nodeName === 'CODE' + } + + function Node (node, options) { node.isBlock = isBlock(node); - node.isCode = node.nodeName.toLowerCase() === 'code' || node.parentNode.isCode; + node.isCode = node.nodeName === 'CODE' || node.parentNode.isCode; node.isBlank = isBlank(node); - node.flankingWhitespace = flankingWhitespace(node); + node.flankingWhitespace = flankingWhitespace(node, options); return node } @@ -656,28 +671,39 @@ ) } - function flankingWhitespace (node) { - var leading = ''; - var trailing = ''; + function flankingWhitespace (node, options) { + if (node.isBlock || (options.preformattedCode && node.isCode)) { + return { leading: '', trailing: '' } + } - if (!node.isBlock) { - var hasLeading = /^\s/.test(node.textContent); - var hasTrailing = /\s$/.test(node.textContent); - var blankWithSpaces = node.isBlank && hasLeading && hasTrailing; + var edges = edgeWhitespace(node.textContent); - if (hasLeading && !isFlankedByWhitespace('left', node)) { - leading = ' '; - } + // abandon leading ASCII WS if left-flanked by ASCII WS + if (edges.leadingAscii && isFlankedByWhitespace('left', node, options)) { + edges.leading = edges.leadingNonAscii; + } - if (!blankWithSpaces && hasTrailing && !isFlankedByWhitespace('right', node)) { - trailing = ' '; - } + // abandon trailing ASCII WS if right-flanked by ASCII WS + if (edges.trailingAscii && isFlankedByWhitespace('right', node, options)) { + edges.trailing = edges.trailingNonAscii; } - return { leading: leading, trailing: trailing } + return { leading: edges.leading, trailing: edges.trailing } } - function isFlankedByWhitespace (side, node) { + function edgeWhitespace (string) { + var m = string.match(/^(([ \t\r\n]*)(\s*))[\s\S]*?((\s*?)([ \t\r\n]*))$/); + return { + leading: m[1], // whole string for whitespace-only strings + leadingAscii: m[2], + leadingNonAscii: m[3], + trailing: m[4], // empty for whitespace-only strings + trailingNonAscii: m[5], + trailingAscii: m[6] + } + } + + function isFlankedByWhitespace (side, node, options) { var sibling; var regExp; var isFlanked; @@ -693,6 +719,8 @@ if (sibling) { if (sibling.nodeType === 3) { isFlanked = regExp.test(sibling.nodeValue); + } else if (options.preformattedCode && sibling.nodeName === 'CODE') { + isFlanked = false; } else if (sibling.nodeType === 1 && !isBlock(sibling)) { isFlanked = regExp.test(sibling.textContent); } @@ -701,8 +729,6 @@ } var reduce = Array.prototype.reduce; - var leadingNewLinesRegExp = /^\n*/; - var trailingNewLinesRegExp = /\n*$/; var escapes = [ [/\\/g, '\\\\'], [/\*/g, '\\*'], @@ -734,6 +760,7 @@ linkStyle: 'inlined', linkReferenceStyle: 'full', br: ' ', + preformattedCode: false, blankReplacement: function (content, node) { return node.isBlock ? '\n\n' : '' }, @@ -766,7 +793,7 @@ if (input === '') return '' - var output = process.call(this, new RootNode(input)); + var output = process.call(this, new RootNode(input, this.options)); return postProcess.call(this, output) }, @@ -855,7 +882,7 @@ function process (parentNode) { var self = this; return reduce.call(parentNode.childNodes, function (output, node) { - node = new Node(node); + node = new Node(node, self.options); var replacement = ''; if (node.nodeType === 3) { @@ -908,31 +935,21 @@ } /** - * Determines the new lines between the current output and the replacement + * Joins replacement to the current output with appropriate number of new lines * @private * @param {String} output The current conversion output * @param {String} replacement The string to append to the output - * @returns The whitespace to separate the current output and the replacement + * @returns Joined output * @type String */ - function separatingNewlines (output, replacement) { - var newlines = [ - output.match(trailingNewLinesRegExp)[0], - replacement.match(leadingNewLinesRegExp)[0] - ].sort(); - var maxNewlines = newlines[newlines.length - 1]; - return maxNewlines.length < 2 ? maxNewlines : '\n\n' - } - - function join (string1, string2) { - var separator = separatingNewlines(string1, string2); - - // Remove trailing/leading newlines and replace with separator - string1 = string1.replace(trailingNewLinesRegExp, ''); - string2 = string2.replace(leadingNewLinesRegExp, ''); + function join (output, replacement) { + var s1 = trimTrailingNewlines(output); + var s2 = trimLeadingNewlines(replacement); + var nls = Math.max(output.length - s1.length, replacement.length - s2.length); + var separator = '\n\n'.substring(0, nls); - return string1 + separator + string2 + return s1 + separator + s2 } /** From eaf80b731728e33d511686421481988bf759ac6d Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 1 Jul 2021 18:44:50 +0530 Subject: [PATCH 49/79] update changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18fd928a0..d74a37dc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ * display actual FLoC header in checkup test * WIP check for `server_tokens` settings (needs work for HTTP2) * nicer output in error/warning tables + * more tests for Cross-Origin-Opener-Policy headers * form templates * guard against a type error in `getAccessKeys` * guard against invalid or malicious input when constructing media-tags for embedding in markdown @@ -26,7 +27,7 @@ * conversions * convert app * some basic office formats - * rich text => markdown + * rich text => markdown (via turndown.js v7.1.1) * handle some pecularities with headings and ids * forms => .csv * trello import with some loss @@ -35,7 +36,6 @@ * catch errors thrown by the diff applier * don't bother returning the hash of a pin list to the client, since they don't use it - # 4.7.0 ## Goals From 29d2fb38efc9b7a29cac0820616a08c8d30ab391 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 2 Jul 2021 12:41:34 +0530 Subject: [PATCH 50/79] trim leading and trailing whitespace from usernames when registering --- www/register/main.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/www/register/main.js b/www/register/main.js index 4c63e4482..4d9274253 100644 --- a/www/register/main.js +++ b/www/register/main.js @@ -47,7 +47,11 @@ define([ var I_REALLY_WANT_TO_USE_MY_EMAIL_FOR_MY_USERNAME = false; var registerClick = function () { - var uname = $uname.val(); + var uname = $uname.val().trim(); + // trim whitespace surrounding the username since it is otherwise included in key derivation + // most people won't realize that its presence is significant + $uname.val(uname); + var passwd = $passwd.val(); var confirmPassword = $confirm.val(); From 09cbeefd99c913043b4de0a2bf7a69931082cc12 Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 2 Jul 2021 14:48:24 +0530 Subject: [PATCH 51/79] double-check that blocks have been written even if 'write' calls back with no error also update the changelog --- CHANGELOG.md | 3 +++ customize.dist/login.js | 15 ++++++++++++--- www/common/cryptpad-common.js | 11 +++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d74a37dc4..92a38bfb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,9 @@ * guard against domExceptions * catch errors thrown by the diff applier * don't bother returning the hash of a pin list to the client, since they don't use it +* accounts + * trim leading and trailing whitespace from usernames when registering + * double-check that login blocks can be loaded after they have been written without error # 4.7.0 diff --git a/customize.dist/login.js b/customize.dist/login.js index 74e5e86f6..3309190c0 100644 --- a/customize.dist/login.js +++ b/customize.dist/login.js @@ -153,7 +153,7 @@ define([ register: isRegister, }; - var RT, blockKeys, blockHash, Pinpad, rpc, userHash; + var RT, blockKeys, blockHash, blockUrl, Pinpad, rpc, userHash; nThen(function (waitFor) { // derive a predefined number of bytes from the user's inputs, @@ -171,7 +171,7 @@ define([ // the rest of their data // determine where a block for your set of keys would be stored - var blockUrl = Block.getBlockUrl(res.opt.blockKeys); + blockUrl = Block.getBlockUrl(res.opt.blockKeys); // Check whether there is a block at that location Util.fetch(blockUrl, waitFor(function (err, block) { @@ -412,12 +412,21 @@ define([ toPublish.edPublic = RT.proxy.edPublic; var blockRequest = Block.serialize(JSON.stringify(toPublish), res.opt.blockKeys); - rpc.writeLoginBlock(blockRequest, waitFor(function (e) { if (e) { console.error(e); + waitFor.abort(); return void cb(e); } + })); + }).nThen(function (waitFor) { + // confirm that the block was actually written before considering registration successful + Util.fetch(blockUrl, waitFor(function (err /*, block */) { + if (err) { + console.error(err); + waitFor.abort(); + return void cb(err); + } console.log("blockInfo available at:", blockHash); LocalStore.setBlockHash(blockHash); diff --git a/www/common/cryptpad-common.js b/www/common/cryptpad-common.js index 9eb5398e7..15234c35e 100644 --- a/www/common/cryptpad-common.js +++ b/www/common/cryptpad-common.js @@ -1937,6 +1937,17 @@ define([ waitFor.abort(); return void cb(obj); } + })); + }).nThen(function (waitFor) { + var blockUrl = Block.getBlockUrl(blockKeys); + Util.fetch(blockUrl, waitFor(function (err /* block */) { + if (err) { + console.error(err); + waitFor.abort(); + return cb({ + error: err, + }); + } console.log("new login block written"); var newBlockHash = Block.getBlockHash(blockKeys); LocalStore.setBlockHash(newBlockHash); From 27f3223490ba132cd5ed995e310790d88be5ad2c Mon Sep 17 00:00:00 2001 From: ansuz Date: Fri, 2 Jul 2021 17:55:15 +0530 Subject: [PATCH 52/79] provide the language we detect to CKEditor --- www/pad/inner.js | 1 + 1 file changed, 1 insertion(+) diff --git a/www/pad/inner.js b/www/pad/inner.js index ae22af23c..304e9abe4 100644 --- a/www/pad/inner.js +++ b/www/pad/inner.js @@ -1327,6 +1327,7 @@ define([ $(waitFor()); }).nThen(function(waitFor) { Ckeditor.config.toolbarCanCollapse = true; + Ckeditor.config.language = Messages._getLanguage(); if (screen.height < 800) { Ckeditor.config.toolbarStartupExpanded = false; $('meta[name=viewport]').attr('content', From ca1016ad3a674b760bbf54767d7b2ad0add6fe97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Benqu=C3=A9?= Date: Fri, 2 Jul 2021 13:45:20 +0100 Subject: [PATCH 53/79] Add spacing to Form results --- www/form/app-form.less | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/www/form/app-form.less b/www/form/app-form.less index 433316448..95802e18b 100644 --- a/www/form/app-form.less +++ b/www/form/app-form.less @@ -470,7 +470,11 @@ //padding: 10px; } + .cp-form-creator-results-export { + margin-bottom: 20px; + } .cp-form-creator-results-content { + padding-bottom: 100px; .cp-form-block { background: @cp_form-bg1; padding: 10px; From bca1c086534cac6fcbbd2e187fa94b3be9253a5c Mon Sep 17 00:00:00 2001 From: yflory Date: Fri, 2 Jul 2021 18:46:03 +0200 Subject: [PATCH 54/79] Fix team pending owner issues --- www/common/outer/team.js | 162 +++++++++++++++++++++++---------------- www/teams/inner.js | 32 ++++---- 2 files changed, 113 insertions(+), 81 deletions(-) diff --git a/www/common/outer/team.js b/www/common/outer/team.js index 0ec224984..3dc39c6f3 100644 --- a/www/common/outer/team.js +++ b/www/common/outer/team.js @@ -986,47 +986,60 @@ define([ var state = team.roster.getState() || {}; var members = state.members || {}; - // Get pending owners - var md = team.listmap.metadata || {}; - if (Array.isArray(md.pending_owners)) { - // Get the members associated to the pending_owners' edPublic and mark them as such - md.pending_owners.forEach(function (ed) { - var member; - Object.keys(members).some(function (curve) { - if (members[curve].edPublic === ed) { - member = members[curve]; - return true; - } - }); - if (!member && teamData.owner) { - var removeOwnership = function (chan) { - ctx.Store.setPadMetadata(null, { - channel: chan, - command: 'RM_PENDING_OWNERS', - value: [ed], - }, function () {}); - }; - removeOwnership(teamData.channel); - removeOwnership(Util.find(teamData, ['keys', 'roster', 'channel'])); - removeOwnership(Util.find(teamData, ['keys', 'chat', 'channel'])); + var md; + nThen(function (waitFor) { + // Get pending owners + ctx.Store.getPadMetadata(null, { + channel: teamData.channel + }, waitFor(function (obj) { + if (obj && obj.error) { + md = team.listmap.metadata || {}; return; } - member.pendingOwner = true; - }); - } + md = obj; + })); + }).nThen(function () { + ctx.pending_owners = md.pending_owners; + if (Array.isArray(md.pending_owners)) { + // Get the members associated to the pending_owners' edPublic and mark them as such + md.pending_owners.forEach(function (ed) { + var member; + Object.keys(members).some(function (curve) { + if (members[curve].edPublic === ed) { + member = members[curve]; + return true; + } + }); + if (!member && teamData.owner) { + var removeOwnership = function (chan) { + ctx.Store.setPadMetadata(null, { + channel: chan, + command: 'RM_PENDING_OWNERS', + value: [ed], + }, function () {}); + }; + removeOwnership(teamData.channel); + removeOwnership(Util.find(teamData, ['keys', 'roster', 'channel'])); + removeOwnership(Util.find(teamData, ['keys', 'chat', 'channel'])); + return; + } + member.pendingOwner = true; + }); + } - // Add online status (using messenger data) - if (ctx.store.messenger) { - var chatData = team.getChatData(); - var online = ctx.store.messenger.getOnlineList(chatData.channel) || []; - online.forEach(function (curve) { - if (members[curve]) { - members[curve].online = true; - } - }); - } + // Add online status (using messenger data) + if (ctx.store.messenger) { + var chatData = team.getChatData(); + var online = ctx.store.messenger.getOnlineList(chatData.channel) || []; + online.forEach(function (curve) { + if (members[curve]) { + members[curve].online = true; + } + }); + } - cb(members); + cb(members); + }); }; // Return folders with edit rights available to everybody (decrypted pad href) @@ -1144,8 +1157,7 @@ define([ if (!teamData) { return void cb ({error: 'ENOENT'}); } var team = ctx.teams[teamId]; if (!team) { return void cb ({error: 'ENOENT'}); } - var md = team.listmap.metadata || {}; - var isPendingOwner = (md.pending_owners || []).indexOf(user.edPublic) !== -1; + var isPendingOwner = user.pendingOwner; nThen(function (waitFor) { var cmd = isPendingOwner ? 'RM_PENDING_OWNERS' : 'RM_OWNERS'; @@ -1364,42 +1376,60 @@ define([ var describeUser = function (ctx, data, cId, cb) { var teamId = data.teamId; if (!teamId) { return void cb({error: 'EINVAL'}); } + var teamData = Util.find(ctx, ['store', 'proxy', 'teams', teamId]); var team = ctx.teams[teamId]; - if (!team) { return void cb ({error: 'ENOENT'}); } + if (!teamData || !team) { return void cb ({error: 'ENOENT'}); } if (!team.roster) { return void cb({error: 'NO_ROSTER'}); } if (!data.curvePublic || !data.data) { return void cb({error: 'MISSING_DATA'}); } var state = team.roster.getState(); var user = state.members[data.curvePublic]; - // It it is an ownership revocation, we have to set it in pad metadata first - if (user.role === "OWNER" && data.data.role !== "OWNER") { - revokeOwnership(ctx, teamId, user, function (err) { - if (!err) { return void cb(); } - console.error(err); - return void cb({error: err}); - }); - return; - } + var md; + nThen(function (waitFor) { + // Get pending owners + ctx.Store.getPadMetadata(null, { + channel: teamData.channel + }, waitFor(function (obj) { + if (obj && obj.error) { + md = team.listmap.metadata || {}; + return; + } + md = obj; + })); + }).nThen(function () { + user.pendingOwner = Array.isArray(md.pending_owners) && + md.pending_owners.indexOf(user.edPublic) !== -1; - // Viewer to editor - if (user.role === "VIEWER" && data.data.role !== "VIEWER") { - changeEditRights(ctx, teamId, user, true, function (obj) { - return void cb(obj); - }); - } + // It it is an ownership revocation, we have to set it in pad metadata first + if (user.role === "OWNER" && data.data.role !== "OWNER") { + revokeOwnership(ctx, teamId, user, function (err) { + if (!err) { return void cb(); } + console.error(err); + return void cb({error: err}); + }); + return; + } - // Editor to viewer - if (user.role !== "VIEWER" && data.data.role === "VIEWER") { - changeEditRights(ctx, teamId, user, false, function (obj) { - return void cb(obj); - }); - } + // Viewer to editor + if (user.role === "VIEWER" && data.data.role !== "VIEWER") { + changeEditRights(ctx, teamId, user, true, function (obj) { + return void cb(obj); + }); + } - var obj = {}; - obj[data.curvePublic] = data.data; - team.roster.describe(obj, function (err) { - if (err) { return void cb({error: err}); } - cb(); + // Editor to viewer + if (user.role !== "VIEWER" && data.data.role === "VIEWER") { + changeEditRights(ctx, teamId, user, false, function (obj) { + return void cb(obj); + }); + } + + var obj = {}; + obj[data.curvePublic] = data.data; + team.roster.describe(obj, function (err) { + if (err) { return void cb({error: err}); } + cb(); + }); }); }; diff --git a/www/teams/inner.js b/www/teams/inner.js index 4b0532628..7a7cf9116 100644 --- a/www/teams/inner.js +++ b/www/teams/inner.js @@ -768,7 +768,7 @@ define([ $(demote).hide(); describeUser(common, data.curvePublic, { role: role - }, promote); + }, demote); }; if (isMe) { return void UI.confirm(Messages.team_demoteMeConfirm, function (yes) { @@ -901,22 +901,24 @@ define([ $header.append(invite); } - if (me && (me.role !== 'OWNER')) { - var leave = h('button.cp-online.btn.btn-danger', Messages.team_leaveButton); - $(leave).click(function () { - UI.confirm(Messages.team_leaveConfirm, function (yes) { - if (!yes) { return; } - APP.module.execCommand('LEAVE_TEAM', { - teamId: APP.team - }, function (obj) { - if (obj && obj.error) { - return void UI.warn(Messages.error); - } - }); + var leave = h('button.cp-online.btn.btn-danger', Messages.team_leaveButton); + $(leave).click(function () { + if (me && me.role === 'OWNER') { + Messages.team_leaveOwner = "Owners can't leave the team. You can demote yourself if there is at least one other owner."; // XXX + return void UI.alert(Messages.team_leaveOwner); + } + UI.confirm(Messages.team_leaveConfirm, function (yes) { + if (!yes) { return; } + APP.module.execCommand('LEAVE_TEAM', { + teamId: APP.team + }, function (obj) { + if (obj && obj.error) { + return void UI.warn(Messages.error); + } }); }); - $header.append(leave); - } + }); + $header.append(leave); var table = h('button.btn.btn-primary', Messages.teams_table); $(table).click(function (e) { From ffad850434fdd31ee38ffd01be2df05097160b9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Benqu=C3=A9?= Date: Mon, 5 Jul 2021 10:12:51 +0100 Subject: [PATCH 55/79] Use more faded red for danger alert backgrounds --- customize.dist/src/less2/include/colortheme-dark.less | 2 +- customize.dist/src/less2/include/colortheme.less | 2 +- customize.dist/src/less2/include/markdown.less | 7 ++----- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/customize.dist/src/less2/include/colortheme-dark.less b/customize.dist/src/less2/include/colortheme-dark.less index f8ea0c92b..5c1649850 100644 --- a/customize.dist/src/less2/include/colortheme-dark.less +++ b/customize.dist/src/less2/include/colortheme-dark.less @@ -53,7 +53,7 @@ @cryptpad_color_light_blue: #00b7d8; @cryptpad_color_red: #ff1100; @cryptpad_color_red_fade: fade(@cryptpad_color_red, 50%); -@cryptpad_color_red_fader: fade(@cryptpad_color_red, 25%); +@cryptpad_color_red_fader: fade(@cryptpad_color_red, 15%); @cryptpad_color_warn_red: @cryptpad_color_red_fade; @cryptpad_color_dark_red: #9e0000; @cryptpad_color_light_red: #FFD4D4; diff --git a/customize.dist/src/less2/include/colortheme.less b/customize.dist/src/less2/include/colortheme.less index a7b103797..18dcb1c0a 100644 --- a/customize.dist/src/less2/include/colortheme.less +++ b/customize.dist/src/less2/include/colortheme.less @@ -53,7 +53,7 @@ @cryptpad_color_light_blue: #00b7d8; @cryptpad_color_red: #ff1100; @cryptpad_color_red_fade: fade(@cryptpad_color_red, 50%); -@cryptpad_color_red_fader: fade(@cryptpad_color_red, 25%); +@cryptpad_color_red_fader: fade(@cryptpad_color_red, 15%); @cryptpad_color_warn_red: @cryptpad_color_red_fade; @cryptpad_color_dark_red: #9e0000; @cryptpad_color_light_red: #FFD4D4; diff --git a/customize.dist/src/less2/include/markdown.less b/customize.dist/src/less2/include/markdown.less index f42a27146..43732187a 100644 --- a/customize.dist/src/less2/include/markdown.less +++ b/customize.dist/src/less2/include/markdown.less @@ -156,14 +156,11 @@ } div.cp-inline-img-warning { - @cryptpad_test_red_fader: fade(@cryptpad_color_red, 15%); // XXX display: inline-block; padding: 10px; - - color: @cryptpad_text_col; // XXX - background-color: @cryptpad_test_red_fader; // XXX @cryptpad_color_red_fader; + color: @cryptpad_text_col; + background-color: @cryptpad_color_red_fader; border: 1px solid @cryptpad_color_red; - .cp-inline-img { display: flex; margin-bottom: 10px; From 15fc16fca33d84362aaebaec0d0f5d6766e953bf Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 5 Jul 2021 18:49:54 +0530 Subject: [PATCH 56/79] update changelog and version strings for 4.8.0 --- CHANGELOG.md | 94 +++++++++++++++++++++++------------------ customize.dist/pages.js | 2 +- package-lock.json | 2 +- package.json | 2 +- 4 files changed, 57 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92a38bfb1..01bb874c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,43 +1,57 @@ -# WIP - -* WIP file conversion utilities -* server - * `installMethod: 'unspecified'` in the default config to distinguish docker installs - * `instancePurpose` on admin panel - * add support for archiving pin lists (instead of deleting them) - * blocks - * archive blocks instead of deleting them outright - * implement as storage API - * validate blocks in worker - * don't include adminEmail in telemetry unless we have consentToContact - * don't include instancePurpose in telemetry if it is "noanswer" -* display warnings when remote resources are blocked - * in code preview -* restrict style tags to a scope when rendering them in markdown preview by compiling their content as scoped less -* iPhone/Safari calendar and notification fixes (data parsing errors) -* checkup - * display actual FLoC header in checkup test - * WIP check for `server_tokens` settings (needs work for HTTP2) - * nicer output in error/warning tables - * more tests for Cross-Origin-Opener-Policy headers -* form templates -* guard against a type error in `getAccessKeys` -* guard against invalid or malicious input when constructing media-tags for embedding in markdown -* Japanese translation -* conversions - * convert app - * some basic office formats - * rich text => markdown (via turndown.js v7.1.1) - * handle some pecularities with headings and ids - * forms => .csv - * trello import with some loss -* fix rendering issue for legacy markdown media tag syntax - * guard against domExceptions - * catch errors thrown by the diff applier -* don't bother returning the hash of a pin list to the client, since they don't use it -* accounts - * trim leading and trailing whitespace from usernames when registering - * double-check that login blocks can be loaded after they have been written without error +# 4.8.0 + +## Goals + +This release cycle we decided to give people a chance to try our forms app and provide feedback before we begin developing its second round of major features and improvements. In the meantime we planned to work mostly on the activities of our [NGI DAPSI](https://dapsi.ngi.eu/) project which concerns client-side file format conversions. Otherwise, we dedicated some of our independently funded time towards some internal code review and security best-practices as a follow-up to the recent quick-scan performed by [Radically Open Security](https://radicallyopensecurity.com/) that was funded by [NLnet](https://nlnet.nl) as a part of our now-closing _CryptPad for Communities_ project. + +## Update notes + +We are still accepting feedback concerning our Form application via [a form hosted on CryptPad.fr](https://cryptpad.fr/form/#/2/form/view/gYs4QS7DetInCXy0z2CQoUW6CwN6kaR2utGsftDzp58/). We will accept feedback here until July 12th, 2021, so if you'd like your opinions to be represented in the app's second round of development act quickly! + +Following our last release we sent out an email to the admins of each outdated instance that had included their addresses in the server's daily telemetry. This appears to have been successful, as more than half of the 700+ instances that provide this telemetry are now running **4.7.0**. Previously, only 15% of instances were running the latest version. It's worth noting that of those admins that are hosting the latest version, less than 10% have opted into future emails warning them of security issues. In case you missed it, this can be done on the admin panel's _Network_ tab. Unlike most companies, we consider excess data collection a liability rather than an asset. As such, adminstrator emails are no longer included in server telemetry unless the admin has consented to be contacted. + +The same HTTP request that communicates server telemetry will soon begin responding with the URL of our latest release notes if it is detected that the remote instance is running an older version. The admin panel's _Network_ tab for instances running 4.7.0 or later will begin prompting admins to view the release notes and update once 4.8.0 is available. + +The Network tab now includes a multiple choice form as well. If you have not disabled your instance's telemetry you can use this field to answer _why you run your instance_ (for a business, an academic institution, personal use, etc.). We intend to use this data to inform our development roadmap, though as always, the fastest way to get us to prioritize your needs is to contact us for a support contract (sales@cryptpad.fr). + +Server telemetry will also include an `installMethod` property. By default this is `"unspecified"`, but we are planning to work with packagers of alternate install methods to modify this property in their installation scripts. This will help us assess what proportion of instances are installed via the steps included in our installation guide vs other methods such as the various docker images. We hope that it will also allow us to determine the source of some common misconfigurations so we can propose some improvements to the root cause. + +Getting off the topic of telemetry: two types of data that were previously deleted outright (pin logs and login blocks) are now archived when the client sends a _remove_ command. This provides for the ability to restore old user credentials in cases where users claim that their new credentials do not work following a password change. Some discretion is required in such cases as a user might have intentionally invalidated their old credentials due to shoulder-surfing or the breach of another service's database where they'd reused credentials. Neither of these types of data are currently included in the scripts which evict old data as they are not likely to consume a significant amount of storage space. In any case, CryptPad's data is stored on the filesystem, so it's always possible to remove outdated files by removing them from `cryptpad/data/archive/*` or whatever path you've configured for your archives. + +This release introduces some minor changes to the provided NGINX configuration file to enable support for WebAssembly where it is required for client-side file format conversions. We've added some new tests on the /checkup/ page that determine whether these changes have been applied. This page can be found via a button on the admin panel. + +To update from 4.7.0 to 4.8.0: + +1. Apply the documented NGINX configuration +2. Stop your server +3. Get the latest code with git +4. Install the latest dependencies with `bower update` and `npm i` +5. Restart your server +6. Confirm that your instance is passing all the tests included on the `/checkup/` page + +## Features + +* Those who prefer using tools localized in Japanese can thank [@Suguru](https://mstdn.progressiv.dev/@suguru) for completing the Japanese translation of the platform's text! CryptPad is a fairly big platform with a lot of text to translate, so we really appreciate how much effort went into this. + * While we're on the topic, CryptPad's _Deutsch_ translation is kept up to date largely by a single member of the German Pirate Party (Piratenpartei Deutschland). This is a huge job and we appreciate your work too! + * Anyone else who wishes to give back to the project by doing the same can contribute translations on an ongoing basis through [our Weblate instance](https://weblate.cryptpad.fr/projects/cryptpad/app/). +* We've implemented a new app for file format conversions as a part of our _INTEROFFICE_ project. At this point this page is largely a test-case for the conversion engine that we hope to integrate more tightly into the rest of the platform. It allows users to load a variety of file formats into their browser and convert to any other format that has a defined conversion process from the original format. What's special about this is that files are converted entirely in your browser, unlike other platforms which do so in the cloud and expose their contents in the process. Currently we support conversion between the following formats in every browser that supports modern web standards (ie. not safari): + * XLSX and ODS + * DOCX and ODT and TXT + * PPTX and ODP +* In addition to the /convert/ page which supports office file formats, we also put some time into improving interoperability for our existing apps. We're introducing the ability to export rich text documents as Markdown (via the [turndown](https://github.com/mixmark-io/turndown) library), to import trello's JSON format into our Kanban app (with some loss of attributes because we don't support all the same features), and to export form summaries as CSV files. +* We've added another extension to our customized markdown renderer which replaces markdown images with a warning that CryptPad blocks remote content to prevent malicious users from tracking visitors to certain pages. Such images should already be blocked by our strict use of Content-Security-Policy headers, but this will provide a better indication why images are failing to load on isnstances that are correctly configured and a modest improvement to users' privacy on instances that aren't. +* Up until now it was possible to include style tags in markdown documents, which some of our more advanced users used in order to customize the appearance of their rendered documents. Unfortunately, these styles were not applied strictly to the markdown preview window, but to the page as a whole, making it possible to break the platform's interface (for that pad) through the use of overly broad and powerful style rules. As of this release style tags are now treated as special elements, such that their contents are compiled as [LESS](https://lesscss.org/) within a scope that is only applied to the preview pane. This was intended as a bug fix, but it's included here as a _feature_ because advanced users might see it as such and use it to do neat things. We have no funding for further work in this direction, however, and presently have no intent of providing documentation about this behaviour. +* The checkup page uses some slightly nicer methods of displaying values returned by tests when the expected value of `true` is not returned. Some tests have been revised to return the problematic value instead of `false` when the test fails, since there were some cases where it was not clear why the test was failing, such as when a header was present but duplicated. +* We've made some server requests related to _pinning files_ moderately faster by skipping an expensive calculation and omitting the value it returned. This value was meant to be used as a checksum to ensure that all of a user's documents were included in the list which should be associated with their account, however, clients used a separate command to fetch this checksum. The value provided in response to the other commands was never used by the client. +* We've implemented a system on the client for defining default templates for particular types of documents across an entire instance in addition to the use of documents in the _templates_ section of the users drive (or that of their teams). This is intended more as a generic system for us to reuse throughout the platform's source than an API for instance admins to use. If there is sufficient interest (and funding) from other admins we'll implement this as an instance configuration point. We now provide a _poll_ template to replicate the features of our old poll app which has been deprecated in favour of forms. + +## Bug fixes + +* It was brought to our attention that the registration page was not trimming leading and trailing whitespace from usernames as intended. We've updated the page to do so, however, accounts created with such characters in their username field must enter their credentials exactly as they were at registration time in order to log in. We have no means of detecting such accounts on the server, as usernames are not visible to server admins. We'll consider this behaviour in the future if we introduce an option to change usernames as we do with passwords. +* We now double-check that login blocks (account credentials encrypted with a key derived from a username and password) can be accessed by the client when registering or changing passwords. It should be sufficient to rely on the server to report whether the encrypted credentials were stored successfully when uploading them, but in instances where these resources don't load due to a misbehaving browser extension it's better that we detect it at registration time rather than after the user creates content that will be difficult to access without assistance determining which extension or browser customization is to blame. +* We learned that the Javascript engine used on iOS has trouble parsing an alternative representation of data strings that every other platform seems to handle. This caused calendars to display incorrect data. Because Apple prevents third-party browsers from including their own JavaScript engines this means that users were affected by this Safari bug regardless of whether they used browsers branded as Safari, Firefox, Chrome, or otherwise. +* After some internal review we now guard against a variety of cases where user-crafted input could trigger a DOMException error and prevent a whole page worth of markdown content to fail to render. While there is no impact for users' privacy or security in this bug, a malicious user could exploit it to be annoying. +* Shortly after our last release a user reported being unable to access their account due to a typeError which we were able to [guard against](https://github.com/xwiki-labs/cryptpad/commit/abc9466abe71a76d1d31ef6a3c2c9bba4d2233e4). # 4.7.0 diff --git a/customize.dist/pages.js b/customize.dist/pages.js index b0fd2f20a..18632f6ea 100644 --- a/customize.dist/pages.js +++ b/customize.dist/pages.js @@ -105,7 +105,7 @@ define([ var imprintUrl = AppConfig.imprint && (typeof(AppConfig.imprint) === "boolean" ? '/imprint.html' : AppConfig.imprint); - Pages.versionString = "v4.7.0"; + Pages.versionString = "v4.8.0"; // used for the about menu diff --git a/package-lock.json b/package-lock.json index ab294ec92..4d7992f82 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cryptpad", - "version": "4.7.0", + "version": "4.8.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index f9dfc2c14..9e18f05f0 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cryptpad", "description": "realtime collaborative visual editor with zero knowlege server", - "version": "4.7.0", + "version": "4.8.0", "license": "AGPL-3.0+", "repository": { "type": "git", From 19eddb1af61c68d098bd49e88f50202c42287ef3 Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 5 Jul 2021 18:57:32 +0530 Subject: [PATCH 57/79] grep the changelog for 'the the ' --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01bb874c5..721fa7ff5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -543,7 +543,7 @@ To upgrade from 3.24.0 to 3.25.0: ## Features * This release makes a lot of changes to how content is loaded over the network. - * Most notably, CryptPad now employs a client-side cache based on the the _indexedDB API_. Browsers that support this functionality will opportunistically store messages in a local cache for the next time they need them. This should make a considerable difference in how quickly you're able to load a pad, particularly if you accessing the server over a low-bandwidth network. + * Most notably, CryptPad now employs a client-side cache based on the _indexedDB API_. Browsers that support this functionality will opportunistically store messages in a local cache for the next time they need them. This should make a considerable difference in how quickly you're able to load a pad, particularly if you accessing the server over a low-bandwidth network. * Uploaded files (images, PDFs, etc.) are also cached in a similar way. Once you'd loaded an asset, your client will prefer to load its local copy instead of the server. * We've updated the code for our _full drive backup_ functionality so that it uses the local cache to load files more quickly. In addition to this, backing up the contents of your drive will also populate the cache as though you had loaded your documents in the normal fashion. This cache will persist until it is invalidated (due to the authoritative document having been deleted or had its history trimmed) or until you have logged out. * We've added the ability to configure the maximum size for automatically downloaded files. Any encrypted files that are above this size will instead require manual interaction to begin downloading. Files that are larger than this limit which are already loaded in your cache will still be automatically displayed. @@ -2063,7 +2063,7 @@ Finally, we prioritized the ability to archive files for a period instead of del * Users with existing friends on the platform will run a migration to allow them to share pads with friends directly instead of sending them a link. * they'll receive a notification indicating the title of the pad and who shared it * if you've already added friends on the platform, you can send them pads from the usual "sharing menu" -* Our code editor already offered the ability to set their color theme and highlighting mode, but now those values will be previewed when mousing over the the option in the dropdown. +* Our code editor already offered the ability to set their color theme and highlighting mode, but now those values will be previewed when mousing over the option in the dropdown. * Our slide editor now offers the same theme selection as the code editor * It's now possible to view the history of a shared folder by clicking the history button while viewing the shared folder's contents. From 30955260669d5124b50c064acf4f7f93c988da0d Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 5 Jul 2021 18:59:33 +0530 Subject: [PATCH 58/79] remove some notes that have been addressed --- www/checkup/main.js | 3 +-- www/convert/inner.js | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/www/checkup/main.js b/www/checkup/main.js index 143d1709b..f4a40d11e 100644 --- a/www/checkup/main.js +++ b/www/checkup/main.js @@ -732,7 +732,6 @@ define([ cb(isHTTPS(trimmedUnsafe) && isHTTPS(trimmedSafe)); }); - [ 'sheet', 'presentation', @@ -787,7 +786,7 @@ define([ " header. This information can make it easier for attackers to find and exploit known vulnerabilities. ", ]; - if (family === 'NGINX') { // XXX incorrect instructions for HTTP2. needs a recompile? + if (family === 'NGINX') { // FIXME incorrect instructions for HTTP2. needs a recompile? msg.appendChild(h('span', text.concat([ "This can be addressed by setting ", code("server_tokens off"), diff --git a/www/convert/inner.js b/www/convert/inner.js index 4952b2d6e..cdcf745ca 100644 --- a/www/convert/inner.js +++ b/www/convert/inner.js @@ -48,7 +48,6 @@ define([ debug("x2t mount done"); }; var getX2t = function (cb) { - // XXX require http headers on firefox... require(['/common/onlyoffice/x2t/x2t.js'], function() { // FIXME why does this fail without an access-control-allow-origin header? var x2t = window.Module; x2t.run(); From 5070a751cd4830a5edf94b0c9a8c5a9a7c11b8ba Mon Sep 17 00:00:00 2001 From: ansuz Date: Mon, 5 Jul 2021 19:21:22 +0530 Subject: [PATCH 59/79] style remote image warnings in slides --- www/slide/app-slide.less | 1 + 1 file changed, 1 insertion(+) diff --git a/www/slide/app-slide.less b/www/slide/app-slide.less index 7e518a1c7..ce12b1de3 100644 --- a/www/slide/app-slide.less +++ b/www/slide/app-slide.less @@ -356,6 +356,7 @@ } .markdown_main(); + .markdown_cryptpad(); .markdown_preformatted-code; .markdown_gfm-table(); From aaa18a3feb320ed39051b32b2c59adc873479a67 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 6 Jul 2021 13:23:41 +0530 Subject: [PATCH 60/79] fix stretched images in 'lightbox' preview modal --- customize.dist/src/less2/include/modals-ui-elements.less | 3 +++ 1 file changed, 3 insertions(+) diff --git a/customize.dist/src/less2/include/modals-ui-elements.less b/customize.dist/src/less2/include/modals-ui-elements.less index 92fef68d0..7ec19a699 100644 --- a/customize.dist/src/less2/include/modals-ui-elements.less +++ b/customize.dist/src/less2/include/modals-ui-elements.less @@ -221,6 +221,9 @@ button { line-height: 1.5; } + img { + align-self: center; + } & > iframe { width: 100%; height: 100%; From f652d11ace5cd53f72c245affc31e28f03280cbe Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 6 Jul 2021 14:02:29 +0530 Subject: [PATCH 61/79] don't show the 'remote image warning' for data URLs in markdown --- www/common/diffMarked.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index 52a8cf0a1..99031b336 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -292,6 +292,9 @@ define([ if (typeof(window.URL) === 'undefined') { return false; } try { var url = new URL(href, ApiConfig.httpUnsafeOrigin); + // FIXME data URLs can be quite large, but that should be addressed + // in the source markdown's, not the renderer + if (url.protocol === 'data:') { return true; } var localURL = new URL(ApiConfig.httpUnsafeOrigin); return url.host === localURL.host; } catch (err) { From 1b50efe7acd97f122ec67a7c0478ad9772f507a2 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 6 Jul 2021 11:13:42 +0200 Subject: [PATCH 62/79] Add more debugging metadata to support tickets --- www/common/outer/team.js | 7 +++++++ www/support/inner.js | 14 +++++++++++--- www/support/ui.js | 14 ++++++++------ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/www/common/outer/team.js b/www/common/outer/team.js index 3dc39c6f3..4184b8482 100644 --- a/www/common/outer/team.js +++ b/www/common/outer/team.js @@ -2040,9 +2040,16 @@ define([ if (['drive', 'teams', 'settings'].indexOf(app) !== -1) { safe = true; } Object.keys(teams).forEach(function (id) { if (!ctx.teams[id]) { return; } + var proxy = ctx.teams[id].proxy || {}; + var nPads = proxy.drive && Object.keys(proxy.drive.filesData || {}).length; + var nSf = proxy.drive && Object.keys(proxy.drive.sharedFolders || {}).length; t[id] = { owner: teams[id].owner, name: teams[id].metadata.name, + channel: teams[id].channel, + numberPads: nPads, + numberSf: nSf, + roster: Util.find(teams[id], ['keys', 'roster', 'channel']), edPublic: Util.find(teams[id], ['keys', 'drive', 'edPublic']), avatar: Util.find(teams[id], ['metadata', 'avatar']), viewer: !Util.find(teams[id], ['keys', 'drive', 'edPrivate']), diff --git a/www/support/inner.js b/www/support/inner.js index f466e7693..6d9e5eae8 100644 --- a/www/support/inner.js +++ b/www/support/inner.js @@ -272,19 +272,27 @@ define([ APP.$rightside = $('
    ', {id: 'cp-sidebarlayout-rightside'}).appendTo(APP.$container); var sFrameChan = common.getSframeChannel(); sFrameChan.onReady(waitFor()); + }).nThen(function (waitFor) { + metadataMgr = common.getMetadataMgr(); + privateData = metadataMgr.getPrivateData(); common.getPinUsage(null, waitFor(function (err, data) { if (err) { return void console.error(err); } APP.pinUsage = data; })); + APP.teamsUsage = {}; + Object.keys(privateData.teams).forEach(function (teamId) { + common.getPinUsage(teamId, waitFor(function (err, data) { + if (err) { return void console.error(err); } + APP.teamsUsage[teamId] = data; + })); + }); }).nThen(function (/*waitFor*/) { createToolbar(); - metadataMgr = common.getMetadataMgr(); - privateData = metadataMgr.getPrivateData(); common.setTabTitle(Messages.supportPage); APP.origin = privateData.origin; APP.readOnly = privateData.readOnly; - APP.support = Support.create(common, false, APP.pinUsage); + APP.support = Support.create(common, false, APP.pinUsage, APP.teamsUsage); // Content var $rightside = APP.$rightside; diff --git a/www/support/ui.js b/www/support/ui.js index 679776c59..90fd43fa5 100644 --- a/www/support/ui.js +++ b/www/support/ui.js @@ -31,9 +31,7 @@ define([ if (typeof(ctx.pinUsage) === 'object') { // pass pin.usage, pin.limit, and pin.plan if supplied - Object.keys(ctx.pinUsage).forEach(function (k) { - data.sender[k] = ctx.pinUsage[k]; - }); + data.sender.quota = ctx.pinUsage; } data.id = id; @@ -45,11 +43,14 @@ define([ data.sender.blockLocation = privateData.blockLocation || ''; data.sender.teams = Object.keys(teams).map(function (key) { var team = teams[key]; - if (!teams) { return; } + if (!team) { return; } var ret = {}; - ['edPublic', 'owner', 'viewer', 'hasSecondaryKey', 'validKeys'].forEach(function (k) { + ['channel', 'roster', 'numberPads', 'numberSf', 'edPublic', 'curvePublic', 'owner', 'viewer', 'hasSecondaryKey', 'validKeys'].forEach(function (k) { ret[k] = team[k]; }); + if (ctx.teamsUsage && ctx.teamsUsage[key]) { + ret.quota = ctx.teamsUsage[key]; + } return ret; }).filter(Boolean); @@ -430,12 +431,13 @@ define([ ]); }; - var create = function (common, isAdmin, pinUsage) { + var create = function (common, isAdmin, pinUsage, teamsUsage) { var ui = {}; var ctx = { common: common, isAdmin: isAdmin, pinUsage: pinUsage || false, + teamsUsage: teamsUsage || false, adminKeys: Array.isArray(ApiConfig.adminKeys)? ApiConfig.adminKeys.slice(): [], }; From bceff56d0245a341de120c0107152c5ae654d33e Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 6 Jul 2021 15:00:54 +0530 Subject: [PATCH 63/79] remove hardcoded translation --- www/teams/inner.js | 1 - 1 file changed, 1 deletion(-) diff --git a/www/teams/inner.js b/www/teams/inner.js index 7a7cf9116..373a27998 100644 --- a/www/teams/inner.js +++ b/www/teams/inner.js @@ -904,7 +904,6 @@ define([ var leave = h('button.cp-online.btn.btn-danger', Messages.team_leaveButton); $(leave).click(function () { if (me && me.role === 'OWNER') { - Messages.team_leaveOwner = "Owners can't leave the team. You can demote yourself if there is at least one other owner."; // XXX return void UI.alert(Messages.team_leaveOwner); } UI.confirm(Messages.team_leaveConfirm, function (yes) { From 7bdabb5cbc3853090b3b133587e820e48495b75d Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 6 Jul 2021 15:42:37 +0530 Subject: [PATCH 64/79] archive blocks before overwriting them --- lib/storage/block.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/storage/block.js b/lib/storage/block.js index d3e7a1869..1078f6d2e 100644 --- a/lib/storage/block.js +++ b/lib/storage/block.js @@ -72,9 +72,16 @@ Block.write = function (Env, publicKey, buffer, _cb) { w.abort(); cb(err); })); + }).nThen(function (w) { + Block.archive(Env, publicKey, w(function (/* err */) { + /* + we proceed even if there are errors. + it might be ENOENT (there is no file to archive) + or EACCES (bad filesystem permissions for the existing archived block?) + or lots of other things, none of which justify preventing the write + */ + })); }).nThen(function () { - // XXX BLOCK check whether this overwrites a block - // XXX archive the old one if so Fs.writeFile(path, buffer, { encoding: 'binary' }, cb); }); }; From b2ed8f4fb0ff94626b253bdd7d1e91a2dd1ef8d0 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 6 Jul 2021 15:44:10 +0530 Subject: [PATCH 65/79] handle missing languages and other errors ...when checking translations for HTML --- scripts/find-html-translations.js | 15 ++++++++++----- scripts/pin-test.js | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 scripts/pin-test.js diff --git a/scripts/find-html-translations.js b/scripts/find-html-translations.js index dabdbac4b..fc1d4a18e 100644 --- a/scripts/find-html-translations.js +++ b/scripts/find-html-translations.js @@ -9,6 +9,7 @@ var simpleTags = [ // FIXME "", + '', '

    ', '

    ', @@ -70,7 +71,7 @@ processLang(EN, 'en', true); [ 'ar', - 'bn_BD', + //'bn_BD', 'ca', 'de', 'es', @@ -86,11 +87,15 @@ processLang(EN, 'en', true); 'ro', 'ru', 'sv', - 'te', + //'te', 'tr', 'zh', ].forEach(function (lang) { - var map = require("../www/common/translations/messages." + lang + ".json"); - if (!Object.keys(map).length) { return; } - processLang(map, lang); + try { + var map = require("../www/common/translations/messages." + lang + ".json"); + if (!Object.keys(map).length) { return; } + processLang(map, lang); + } catch (err) { + console.error(err); + } }); diff --git a/scripts/pin-test.js b/scripts/pin-test.js new file mode 100644 index 000000000..fe313c45b --- /dev/null +++ b/scripts/pin-test.js @@ -0,0 +1,19 @@ +var Pins = require("../lib/pins"); +var Fs = require("fs"); + +var content = Fs.readFileSync('./data/pins/Bp/BpL3pEyX2IlfsvxQELB9uz5qh+40re0gD6J6LOobBm8=.ndjson', 'utf8'); + +//var lines = content.split("\n"); + +//console.log(content); + +var result; + +for (var i = 0; i < 10000; i++) { + result = Pins.calculateFromLog(content, function (label, data) { + console.log([label, data]); + }); +} + +//console.log(result, result.length); + From 15dc966f50aefa8c2f968a8b31ec724779fdba41 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 6 Jul 2021 15:47:34 +0530 Subject: [PATCH 66/79] remove local benchmarking test with hardcoded values --- scripts/pin-test.js | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 scripts/pin-test.js diff --git a/scripts/pin-test.js b/scripts/pin-test.js deleted file mode 100644 index fe313c45b..000000000 --- a/scripts/pin-test.js +++ /dev/null @@ -1,19 +0,0 @@ -var Pins = require("../lib/pins"); -var Fs = require("fs"); - -var content = Fs.readFileSync('./data/pins/Bp/BpL3pEyX2IlfsvxQELB9uz5qh+40re0gD6J6LOobBm8=.ndjson', 'utf8'); - -//var lines = content.split("\n"); - -//console.log(content); - -var result; - -for (var i = 0; i < 10000; i++) { - result = Pins.calculateFromLog(content, function (label, data) { - console.log([label, data]); - }); -} - -//console.log(result, result.length); - From 706166bc60913a01f94bc55ed0d411458a36d665 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 6 Jul 2021 16:05:57 +0530 Subject: [PATCH 67/79] add last-minute fixes to the changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 721fa7ff5..5c05ce5ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ To update from 4.7.0 to 4.8.0: * The checkup page uses some slightly nicer methods of displaying values returned by tests when the expected value of `true` is not returned. Some tests have been revised to return the problematic value instead of `false` when the test fails, since there were some cases where it was not clear why the test was failing, such as when a header was present but duplicated. * We've made some server requests related to _pinning files_ moderately faster by skipping an expensive calculation and omitting the value it returned. This value was meant to be used as a checksum to ensure that all of a user's documents were included in the list which should be associated with their account, however, clients used a separate command to fetch this checksum. The value provided in response to the other commands was never used by the client. * We've implemented a system on the client for defining default templates for particular types of documents across an entire instance in addition to the use of documents in the _templates_ section of the users drive (or that of their teams). This is intended more as a generic system for us to reuse throughout the platform's source than an API for instance admins to use. If there is sufficient interest (and funding) from other admins we'll implement this as an instance configuration point. We now provide a _poll_ template to replicate the features of our old poll app which has been deprecated in favour of forms. +* We've included some more non-sensitive information about users' teams to the debugging data to which is automatically submitted along with support tickets, such as the id of the team's drive, roster, and how large the drive's contents are. ## Bug fixes @@ -52,6 +53,8 @@ To update from 4.7.0 to 4.8.0: * We learned that the Javascript engine used on iOS has trouble parsing an alternative representation of data strings that every other platform seems to handle. This caused calendars to display incorrect data. Because Apple prevents third-party browsers from including their own JavaScript engines this means that users were affected by this Safari bug regardless of whether they used browsers branded as Safari, Firefox, Chrome, or otherwise. * After some internal review we now guard against a variety of cases where user-crafted input could trigger a DOMException error and prevent a whole page worth of markdown content to fail to render. While there is no impact for users' privacy or security in this bug, a malicious user could exploit it to be annoying. * Shortly after our last release a user reported being unable to access their account due to a typeError which we were able to [guard against](https://github.com/xwiki-labs/cryptpad/commit/abc9466abe71a76d1d31ef6a3c2c9bba4d2233e4). +* Images appearing in the 'lightbox' preview modal no longer appear stretched. +* Before applying actions that modify the team's membership we now confirm that server-enforced permissions match our local state. # 4.7.0 From f13b82bdf6fecc39d514aee41f94a68069ec9d5d Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 6 Jul 2021 16:14:40 +0530 Subject: [PATCH 68/79] disable integrated eviction by default we'll implement an admin panel checkbox to enable it later --- lib/env.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/env.js b/lib/env.js index 141e028ae..3d58f11f7 100644 --- a/lib/env.js +++ b/lib/env.js @@ -123,7 +123,7 @@ module.exports.create = function (config) { maxWorkers: config.maxWorkers, disableIntegratedTasks: config.disableIntegratedTasks || false, - disableIntegratedEviction: config.disableIntegratedEviction || true, // XXX false, + disableIntegratedEviction: typeof(config.disableIntegratedEviction) === 'undefined'? true: config.disableIntegratedEviction, // XXX false, lastEviction: +new Date(), evictionReport: {}, commandTimers: {}, From 7049396bd958f06c5ab1111e665e2fa465560d54 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 6 Jul 2021 16:14:52 +0530 Subject: [PATCH 69/79] remove unneeded XXX note --- www/common/application_config_internal.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/common/application_config_internal.js b/www/common/application_config_internal.js index 971da32ee..15e5b5e34 100644 --- a/www/common/application_config_internal.js +++ b/www/common/application_config_internal.js @@ -22,7 +22,7 @@ define(function() { */ AppConfig.registeredOnlyTypes = ['file', 'contacts', 'notifications', 'support']; - // XXX to prevent apps that aren't officially supported from showing up + // to prevent apps that aren't officially supported from showing up // in the document creation modal AppConfig.hiddenTypes = ['drive', 'teams', 'contacts', 'todo', 'file', 'accounts', 'calendar', 'poll', 'convert', //'doc', 'presentation' From 59f24344a60b1415ccdf48dcf4eebd7919ed8149 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 6 Jul 2021 17:16:22 +0530 Subject: [PATCH 70/79] add a note to the changelog about imperfect form result CSV export --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c05ce5ac..f69b0d723 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ To update from 4.7.0 to 4.8.0: * DOCX and ODT and TXT * PPTX and ODP * In addition to the /convert/ page which supports office file formats, we also put some time into improving interoperability for our existing apps. We're introducing the ability to export rich text documents as Markdown (via the [turndown](https://github.com/mixmark-io/turndown) library), to import trello's JSON format into our Kanban app (with some loss of attributes because we don't support all the same features), and to export form summaries as CSV files. + * note: the currently merged implementation does not preserve all information for complex question/answer types such as polls, multi-line radios, or multi-line checkboxes. More improvements will be included in our next release. * We've added another extension to our customized markdown renderer which replaces markdown images with a warning that CryptPad blocks remote content to prevent malicious users from tracking visitors to certain pages. Such images should already be blocked by our strict use of Content-Security-Policy headers, but this will provide a better indication why images are failing to load on isnstances that are correctly configured and a modest improvement to users' privacy on instances that aren't. * Up until now it was possible to include style tags in markdown documents, which some of our more advanced users used in order to customize the appearance of their rendered documents. Unfortunately, these styles were not applied strictly to the markdown preview window, but to the page as a whole, making it possible to break the platform's interface (for that pad) through the use of overly broad and powerful style rules. As of this release style tags are now treated as special elements, such that their contents are compiled as [LESS](https://lesscss.org/) within a scope that is only applied to the preview pane. This was intended as a bug fix, but it's included here as a _feature_ because advanced users might see it as such and use it to do neat things. We have no funding for further work in this direction, however, and presently have no intent of providing documentation about this behaviour. * The checkup page uses some slightly nicer methods of displaying values returned by tests when the expected value of `true` is not returned. Some tests have been revised to return the problematic value instead of `false` when the test fails, since there were some cases where it was not clear why the test was failing, such as when a header was present but duplicated. From 663f0d87717a5b348ba559d1fb6fe65981ec281a Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 6 Jul 2021 17:47:13 +0530 Subject: [PATCH 71/79] Display confirmation on "Log out everywhere" in the upper right menu addresses #765 --- www/common/common-ui-elements.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/www/common/common-ui-elements.js b/www/common/common-ui-elements.js index a7ef39de6..0ab010cb5 100644 --- a/www/common/common-ui-elements.js +++ b/www/common/common-ui-elements.js @@ -1886,8 +1886,11 @@ define([ }, content: h('span', Messages.logoutEverywhere), action: function () { - Common.getSframeChannel().query('Q_LOGOUT_EVERYWHERE', null, function () { - Common.gotoURL(origin + '/'); + UI.confirm(Messages.settings_logoutEverywhereConfirm, function (yes) { + if (!yes) { return; } + Common.getSframeChannel().query('Q_LOGOUT_EVERYWHERE', null, function () { + Common.gotoURL(origin + '/'); + }); }); }, }); From 230425c3dea752a902e855a3bb52551baa656a97 Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 6 Jul 2021 17:54:18 +0530 Subject: [PATCH 72/79] update changelog again --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f69b0d723..a7987f731 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ To update from 4.7.0 to 4.8.0: * We've made some server requests related to _pinning files_ moderately faster by skipping an expensive calculation and omitting the value it returned. This value was meant to be used as a checksum to ensure that all of a user's documents were included in the list which should be associated with their account, however, clients used a separate command to fetch this checksum. The value provided in response to the other commands was never used by the client. * We've implemented a system on the client for defining default templates for particular types of documents across an entire instance in addition to the use of documents in the _templates_ section of the users drive (or that of their teams). This is intended more as a generic system for us to reuse throughout the platform's source than an API for instance admins to use. If there is sufficient interest (and funding) from other admins we'll implement this as an instance configuration point. We now provide a _poll_ template to replicate the features of our old poll app which has been deprecated in favour of forms. * We've included some more non-sensitive information about users' teams to the debugging data to which is automatically submitted along with support tickets, such as the id of the team's drive, roster, and how large the drive's contents are. +* The _Log out everywhere_ option that is displayed in the user admin menu in the top-right corner of the page for logged-in users now displays a confirmation before terminating all remote sessions. ## Bug fixes From 9bb884cac6379663c0dc66ba478d80325e5108ec Mon Sep 17 00:00:00 2001 From: ansuz Date: Tue, 6 Jul 2021 18:21:47 +0530 Subject: [PATCH 73/79] fix a type error caused by (my) invalid API use --- lib/commands/block.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/commands/block.js b/lib/commands/block.js index 195e0c494..4af32af70 100644 --- a/lib/commands/block.js +++ b/lib/commands/block.js @@ -88,7 +88,7 @@ Block.validateAncestorProof = function (Env, proof, _cb) { } // else fall through to next step }).nThen(function () { - Block.check(Env, pub, function (err) { + BlockStore.check(Env, pub, function (err) { if (err) { return void cb('E_MISSING_ANCESTOR'); } cb(void 0, pub); }); From 0560a9a403f20e43efeb1b7026620f61451d2a25 Mon Sep 17 00:00:00 2001 From: yflory Date: Tue, 6 Jul 2021 16:09:30 +0200 Subject: [PATCH 74/79] Fix forms CSV export --- www/form/export.js | 25 +++++++++++++++++-------- www/form/inner.js | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/www/form/export.js b/www/form/export.js index 0e78d171a..4afc4170a 100644 --- a/www/form/export.js +++ b/www/form/export.js @@ -18,13 +18,18 @@ define([ var csv = ""; var form = content.form; - var questions = Object.keys(form).map(function (key) { + var questions = [Messages.form_poll_time, Messages.share_formView]; + + content.order.forEach(function (key) { var obj = form[key]; if (!obj) { return; } - return obj.q || Messages.form_default; - }).filter(Boolean); - questions.unshift(Messages.share_formView); // "Participant" - questions.unshift(Messages.form_poll_time); // "Time" + var type = obj.type; + if (!TYPES[type]) { return; } // Ignore static types + var c; + if (TYPES[type] && TYPES[type].exportCSV) { c = TYPES[type].exportCSV(false, obj); } + if (!c) { c = [obj.q || Messages.form_default]; } + Array.prototype.push.apply(questions, c); + }); questions.forEach(function (v, i) { if (i) { csv += ','; } @@ -39,10 +44,14 @@ define([ var user = msg._userdata || {}; csv += escapeCSV(time); csv += ',' + escapeCSV(user.name || Messages.anonymous); - Object.keys(form).forEach(function (key) { + content.order.forEach(function (key) { var type = form[key].type; - if (TYPES[type] && TYPES[type].exportCSV) { - csv += ',' + escapeCSV(TYPES[type].exportCSV(msg[key])); + if (!TYPES[type]) { return; } // Ignore static types + if (TYPES[type].exportCSV) { + var res = TYPES[type].exportCSV(msg[key], form[key]).map(function (str) { + return escapeCSV(str); + }).join(','); + csv += ',' + res; return; } csv += ',' + escapeCSV(String(msg[key] || '')); diff --git a/www/form/inner.js b/www/form/inner.js index 8e6b30d47..3fe71873d 100644 --- a/www/form/inner.js +++ b/www/form/inner.js @@ -1215,6 +1215,20 @@ define([ return h('div.cp-form-results-type-radio', results); }, + exportCSV: function (answer, form) { + var opts = form.opts; + var q = form.q || Messages.form_default; + if (answer === false) { + return (opts.items || []).map(function (obj) { + return q + ' | ' + obj.v; + }); + } + if (!answer) { return ['']; } + return (opts.items || []).map(function (obj) { + var uid = obj.uid; + return String(answer[uid] || ''); + }); + }, icon: h('i.cptools.cptools-form-grid-radio') }, checkbox: { @@ -1429,6 +1443,20 @@ define([ return h('div.cp-form-results-type-radio', results); }, + exportCSV: function (answer, form) { + var opts = form.opts; + var q = form.q || Messages.form_default; + if (answer === false) { + return (opts.items || []).map(function (obj) { + return q + ' | ' + obj.v; + }); + } + if (!answer) { return ['']; } + return (opts.items || []).map(function (obj) { + var uid = obj.uid; + return String(answer[uid] || ''); + }); + }, icon: h('i.cptools.cptools-form-grid-check') }, sort: { @@ -1650,13 +1678,14 @@ define([ return h('div.cp-form-type-poll', lines); }, exportCSV: function (answer) { - if (!answer || !answer.values) { return ''; } + if (answer === false) { return; } + if (!answer || !answer.values) { return ['']; } var str = ''; Object.keys(answer.values).sort().forEach(function (k, i) { if (i !== 0) { str += ';'; } str += k.replace(';', '').replace(':', '') + ':' + answer.values[k]; }); - return str; + return [str]; }, icon: h('i.cptools.cptools-form-poll') }, From fceab00a6bb0ac813cd4132ab41709fd3a3ebbbc Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 7 Jul 2021 13:06:07 +0530 Subject: [PATCH 75/79] guard against type errors when exporting results as CSV and label a hardcoded string --- www/form/inner.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/www/form/inner.js b/www/form/inner.js index 3fe71873d..07cd07ce5 100644 --- a/www/form/inner.js +++ b/www/form/inner.js @@ -1216,7 +1216,7 @@ define([ return h('div.cp-form-results-type-radio', results); }, exportCSV: function (answer, form) { - var opts = form.opts; + var opts = form.opts || {}; var q = form.q || Messages.form_default; if (answer === false) { return (opts.items || []).map(function (obj) { @@ -1444,7 +1444,7 @@ define([ return h('div.cp-form-results-type-radio', results); }, exportCSV: function (answer, form) { - var opts = form.opts; + var opts = form.opts || {}; var q = form.q || Messages.form_default; if (answer === false) { return (opts.items || []).map(function (obj) { @@ -1701,7 +1701,7 @@ define([ var controls = h('div.cp-form-creator-results-controls'); var $controls = $(controls).appendTo($container); - Messages.form_exportCSV = "Export results as CSV"; + Messages.form_exportCSV = "Export results as CSV"; // XXX var exportButton = h('button.btn.btn-secondary', Messages.form_exportCSV); var exportCSV = h('div.cp-form-creator-results-export', exportButton); $(exportCSV).appendTo($container); From e7654112d2281901b45e60b4edc2de9e2e2303c1 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 7 Jul 2021 13:10:07 +0530 Subject: [PATCH 76/79] remove note from changelog about incomplete CSV export --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7987f731..55d0e2534 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,6 @@ To update from 4.7.0 to 4.8.0: * DOCX and ODT and TXT * PPTX and ODP * In addition to the /convert/ page which supports office file formats, we also put some time into improving interoperability for our existing apps. We're introducing the ability to export rich text documents as Markdown (via the [turndown](https://github.com/mixmark-io/turndown) library), to import trello's JSON format into our Kanban app (with some loss of attributes because we don't support all the same features), and to export form summaries as CSV files. - * note: the currently merged implementation does not preserve all information for complex question/answer types such as polls, multi-line radios, or multi-line checkboxes. More improvements will be included in our next release. * We've added another extension to our customized markdown renderer which replaces markdown images with a warning that CryptPad blocks remote content to prevent malicious users from tracking visitors to certain pages. Such images should already be blocked by our strict use of Content-Security-Policy headers, but this will provide a better indication why images are failing to load on isnstances that are correctly configured and a modest improvement to users' privacy on instances that aren't. * Up until now it was possible to include style tags in markdown documents, which some of our more advanced users used in order to customize the appearance of their rendered documents. Unfortunately, these styles were not applied strictly to the markdown preview window, but to the page as a whole, making it possible to break the platform's interface (for that pad) through the use of overly broad and powerful style rules. As of this release style tags are now treated as special elements, such that their contents are compiled as [LESS](https://lesscss.org/) within a scope that is only applied to the preview pane. This was intended as a bug fix, but it's included here as a _feature_ because advanced users might see it as such and use it to do neat things. We have no funding for further work in this direction, however, and presently have no intent of providing documentation about this behaviour. * The checkup page uses some slightly nicer methods of displaying values returned by tests when the expected value of `true` is not returned. Some tests have been revised to return the problematic value instead of `false` when the test fails, since there were some cases where it was not clear why the test was failing, such as when a header was present but duplicated. From 96eb5f614e316d5ba65410149e89061248118ee6 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 7 Jul 2021 14:03:54 +0530 Subject: [PATCH 77/79] fix docs link for remote content and localize it --- www/common/diffMarked.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/www/common/diffMarked.js b/www/common/diffMarked.js index 99031b336..0fe8aed0a 100644 --- a/www/common/diffMarked.js +++ b/www/common/diffMarked.js @@ -9,11 +9,13 @@ define([ '/common/media-tag.js', '/customize/messages.js', '/common/less.min.js', + '/customize/pages.js', + '/common/highlight/highlight.pack.js', '/lib/diff-dom/diffDOM.js', '/bower_components/tweetnacl/nacl-fast.min.js', 'css!/common/highlight/styles/'+ (window.CryptPad_theme === 'dark' ? 'dark.css' : 'github.css') -],function ($, ApiConfig, Marked, Hash, Util, h, MT, MediaTag, Messages, Less) { +],function ($, ApiConfig, Marked, Hash, Util, h, MT, MediaTag, Messages, Less, Pages) { var DiffMd = {}; var Highlight = window.hljs; @@ -344,7 +346,7 @@ define([ ]), h('br'), h('a.cp-learn-more', { - href: 'https://docs.cryptpad.fr/user_guide/security.html#remote-content', + href: Pages.localizeDocsLink('https://docs.cryptpad.fr/en/user_guide/security.html#remote-content'), }, [ Messages.resources_learnWhy ]), From ccddcefc1d2e2ee0f41b05d12c430a346662b573 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 7 Jul 2021 14:16:43 +0530 Subject: [PATCH 78/79] remove hardcoded translation --- www/form/inner.js | 1 - 1 file changed, 1 deletion(-) diff --git a/www/form/inner.js b/www/form/inner.js index 07cd07ce5..b2e303f9c 100644 --- a/www/form/inner.js +++ b/www/form/inner.js @@ -1701,7 +1701,6 @@ define([ var controls = h('div.cp-form-creator-results-controls'); var $controls = $(controls).appendTo($container); - Messages.form_exportCSV = "Export results as CSV"; // XXX var exportButton = h('button.btn.btn-secondary', Messages.form_exportCSV); var exportCSV = h('div.cp-form-creator-results-export', exportButton); $(exportCSV).appendTo($container); From a398af12134c6dd6b6f8525d5d241f321b021d80 Mon Sep 17 00:00:00 2001 From: ansuz Date: Wed, 7 Jul 2021 14:18:48 +0530 Subject: [PATCH 79/79] use existing translation until next release --- www/form/inner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/form/inner.js b/www/form/inner.js index b2e303f9c..2a1501ecb 100644 --- a/www/form/inner.js +++ b/www/form/inner.js @@ -1701,7 +1701,7 @@ define([ var controls = h('div.cp-form-creator-results-controls'); var $controls = $(controls).appendTo($container); - var exportButton = h('button.btn.btn-secondary', Messages.form_exportCSV); + var exportButton = h('button.btn.btn-secondary', Messages.exportButton); // XXX form_exportCSV; var exportCSV = h('div.cp-form-creator-results-export', exportButton); $(exportCSV).appendTo($container); var results = h('div.cp-form-creator-results-content');