From 44fdc334bf0c718beb2261563283dfa526e75517 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 11 Feb 2021 15:03:35 +0530 Subject: [PATCH 1/3] a simple script to identify unused translations --- scripts/unused-translations.js | 82 ++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 scripts/unused-translations.js diff --git a/scripts/unused-translations.js b/scripts/unused-translations.js new file mode 100644 index 000000000..ac925b327 --- /dev/null +++ b/scripts/unused-translations.js @@ -0,0 +1,82 @@ +var Messages = require("../www/common/translations/messages.json"); +var Exec = require("child_process").exec; + +var ignoreLines = function (source, pattern) { + if (!pattern.test(source)) { return source; } + return source.split('\n') + .map(function (line) { + if (pattern.test(line)) { return ''; } + return line; + }) + .filter(Boolean) + .join("\n"); +}; + +var grep = function (pattern, cb) { + var command = 'git grep ' + pattern + " -- ':/' ':(exclude)www/common/translations/*'"; + Exec(command, function (err, stdout, stderr) { + if (err && err.code === 1 && err.killed === false) { + return cb(void 0, true, "NOT_FOUND"); + } + stdout = ignoreLines(stdout, /^CHANGELOG\.md:/); + stdout = ignoreLines(stdout, /^LICENSE:/); + stdout = ignoreLines(stdout, /\/onlyoffice/); + stdout = ignoreLines(stdout, /package.*\.json/); + stdout = ignoreLines(stdout, /package\.json/); + stdout = ignoreLines(stdout, /chainpad\.dist\.js/); + stdout = ignoreLines(stdout, /MathJax\.js/); + stdout = ignoreLines(stdout, /Binary file/); + //stdout = stdout .replace(/^CHANGELOG\.md:.*$/g, '') .replace(/^LICENSE:.*$/g, ''); + + if (err) { + if (err.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER') { + return cb(void 0, true, 'TOO_MUCH'); + } + return void cb(err); + } + if (/data\-localization/.test(stdout)) { + return cb(void 0, true, "DATA_LOCALIZATION"); + } + if (/(Messages|Msg|messages)\./.test(stdout)) { + return cb(void 0, false); + } + if (/_cat_/.test(stdout)) { + return cb(void 0, true, 'POSSIBLE_CATEGORY'); + } + + //console.log(pattern, arguments); + cb(void 0, true, 'OTHER', stdout); + }); +}; + +var keys = Object.keys(Messages); +var total = keys.length; + +var limit = total; + +var next = function () { + var key = keys[0]; + if (!key) { return void console.log("[DONE]"); } + keys.shift(); + + if (!limit--) { return void console.log("[DONE]"); } + + grep(key, function (err, flagged, reason, output) { + if (err) { + return; + console.error("[%s]", key, err); + console.log(); + } else if (!flagged) { + + } else if (reason === 'OTHER') { + console.log('[%s] flagged for [OTHER]', key, output); + console.log(); + } else { + console.log("[%s] flagged for [%s]", key, reason || '???'); + } + + next(); + }); +}; + +next(); From 3424a2c96f1dcbf002de5fcdcaea0009aec76377 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 11 Feb 2021 17:53:26 +0530 Subject: [PATCH 2/3] optimize the search for unused tag and apply better categories --- scripts/unused-translations.js | 57 ++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/scripts/unused-translations.js b/scripts/unused-translations.js index ac925b327..dbcba1b07 100644 --- a/scripts/unused-translations.js +++ b/scripts/unused-translations.js @@ -12,21 +12,55 @@ var ignoreLines = function (source, pattern) { .join("\n"); }; +var GENERATED_PATTERNS = [ + /(admin|settings)_.*(Hint|Title|Button)/, + /settings_colortheme/, + /loading_(state|drive|pad)_/, + /(admin|notifications|support|team|settings)_cat_/, + /features_f/, +]; +var isPossiblyGenerated = function (key) { + return GENERATED_PATTERNS.some(function (patt) { + return patt.test(key); + }); +}; + var grep = function (pattern, cb) { - var command = 'git grep ' + pattern + " -- ':/' ':(exclude)www/common/translations/*'"; + var exclude = [ + 'www/common/translations/*', + 'www/common/onlyoffice/*', + 'www/lib/*', + 'www/common/pdfjs/*', + '*.css', + 'www/common/highlight/*', + '*.min.js', + '.lesshintrc', + 'CHANGELOG.md', + 'LICENSE', + 'package*.json', + 'www/debug/chainpad.dist.js', + 'www/pad/mathjax/MathJax.js', + 'www/common/hyperscript.js', + 'www/common/jscolor.js', + './/scripts/*', + './lib/*', + './docs/*', + ].map(function (patt) { + return "':(exclude)" + patt + "'"; + }).join(' '); + + // grep this repository, ignoring binary files and excluding anything matching the above patterns + var ignoreBinaries= '--binary-files=without-match '; + var command = 'git grep ' + pattern + " -- ':/' " + exclude; + Exec(command, function (err, stdout, stderr) { if (err && err.code === 1 && err.killed === false) { + if (isPossiblyGenerated(pattern)) { + return cb(void 0, true, 'POSSIBLY_GENERATED'); + } return cb(void 0, true, "NOT_FOUND"); } - stdout = ignoreLines(stdout, /^CHANGELOG\.md:/); - stdout = ignoreLines(stdout, /^LICENSE:/); - stdout = ignoreLines(stdout, /\/onlyoffice/); - stdout = ignoreLines(stdout, /package.*\.json/); - stdout = ignoreLines(stdout, /package\.json/); - stdout = ignoreLines(stdout, /chainpad\.dist\.js/); - stdout = ignoreLines(stdout, /MathJax\.js/); stdout = ignoreLines(stdout, /Binary file/); - //stdout = stdout .replace(/^CHANGELOG\.md:.*$/g, '') .replace(/^LICENSE:.*$/g, ''); if (err) { if (err.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER') { @@ -40,9 +74,6 @@ var grep = function (pattern, cb) { if (/(Messages|Msg|messages)\./.test(stdout)) { return cb(void 0, false); } - if (/_cat_/.test(stdout)) { - return cb(void 0, true, 'POSSIBLE_CATEGORY'); - } //console.log(pattern, arguments); cb(void 0, true, 'OTHER', stdout); @@ -56,7 +87,7 @@ var limit = total; var next = function () { var key = keys[0]; - if (!key) { return void console.log("[DONE]"); } + if (!key) { return; } keys.shift(); if (!limit--) { return void console.log("[DONE]"); } From 9a42df3e0802adf50e1cbee6bf8fa6ba8892f240 Mon Sep 17 00:00:00 2001 From: ansuz Date: Thu, 11 Feb 2021 19:49:20 +0530 Subject: [PATCH 3/3] update unused translations script more: * fine-tune search by excluding more search patterns * print more context when sensible * sort keys alphabetically --- scripts/unused-translations.js | 39 ++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/scripts/unused-translations.js b/scripts/unused-translations.js index dbcba1b07..8cb2fbc75 100644 --- a/scripts/unused-translations.js +++ b/scripts/unused-translations.js @@ -39,12 +39,16 @@ var grep = function (pattern, cb) { 'LICENSE', 'package*.json', 'www/debug/chainpad.dist.js', - 'www/pad/mathjax/MathJax.js', + 'www/pad/mathjax/*', 'www/common/hyperscript.js', 'www/common/jscolor.js', './/scripts/*', './lib/*', './docs/*', + './github/*', + '*.svg', + '*.md', + './config/*', ].map(function (patt) { return "':(exclude)" + patt + "'"; }).join(' '); @@ -58,18 +62,18 @@ var grep = function (pattern, cb) { if (isPossiblyGenerated(pattern)) { return cb(void 0, true, 'POSSIBLY_GENERATED'); } - return cb(void 0, true, "NOT_FOUND"); + return cb(void 0, true, "NOT_FOUND", stdout); } stdout = ignoreLines(stdout, /Binary file/); if (err) { if (err.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER') { - return cb(void 0, true, 'TOO_MUCH'); + return cb(void 0, true, 'TOO_MUCH', stdout); } return void cb(err); } if (/data\-localization/.test(stdout)) { - return cb(void 0, true, "DATA_LOCALIZATION"); + return cb(void 0, true, "DATA_LOCALIZATION", stdout); } if (/(Messages|Msg|messages)\./.test(stdout)) { return cb(void 0, false); @@ -80,11 +84,31 @@ var grep = function (pattern, cb) { }); }; -var keys = Object.keys(Messages); +var keys = Object.keys(Messages).sort(); var total = keys.length; var limit = total; +var lineCount = function (s) { + var i = 0; + s.replace(/\n/g, function () { i++; return ''; }); + return i; +}; + +var conditionallyPrintContent = function (output) { + if (!output) { return; } + if (lineCount(output) < 12) { + output.split('\n').map(function (line) { + if (!line) { return; } + console.log('\t> ' + line); + }); + //console.log(output); + console.log(); + } else { + console.log("\t> too much content to print"); + } +}; + var next = function () { var key = keys[0]; if (!key) { return; } @@ -100,10 +124,11 @@ var next = function () { } else if (!flagged) { } else if (reason === 'OTHER') { - console.log('[%s] flagged for [OTHER]', key, output); - console.log(); + console.log('[%s] flagged for [OTHER]', key); + conditionallyPrintContent(output); } else { console.log("[%s] flagged for [%s]", key, reason || '???'); + conditionallyPrintContent(output); } next();