From 9a6d63960be6da069ba65b2b73b9238c378c95fd Mon Sep 17 00:00:00 2001
From: Caleb James DeLisle <cjd@cjdns.fr>
Date: Thu, 11 May 2017 15:54:15 +0200
Subject: [PATCH] Added a script to get the amount of data which is pinned

---
 package.json  |  13 ++++---
 pinneddata.js | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 108 insertions(+), 6 deletions(-)
 create mode 100644 pinneddata.js

diff --git a/package.json b/package.json
index ed76592b6..4b78c8396 100644
--- a/package.json
+++ b/package.json
@@ -3,11 +3,12 @@
   "description": "realtime collaborative visual editor with zero knowlege server",
   "version": "1.5.0",
   "dependencies": {
+    "chainpad-server": "^1.0.1",
     "express": "~4.10.1",
-    "ws": "^1.0.1",
     "nthen": "~0.1.0",
+    "saferphore": "0.0.1",
     "tweetnacl": "~0.12.2",
-    "chainpad-server": "^1.0.1"
+    "ws": "^1.0.1"
   },
   "devDependencies": {
     "jshint": "~2.9.1",
@@ -15,9 +16,9 @@
     "less": "2.7.1"
   },
   "scripts": {
-      "lint": "jshint --config .jshintrc --exclude-path .jshintignore .",
-      "test": "node TestSelenium.js",
-      "style": "lessc ./customize.dist/src/less/cryptpad.less > ./customize.dist/main.css && lessc ./customize.dist/src/less/toolbar.less > ./customize.dist/toolbar.css && lessc ./www/drive/file.less > ./www/drive/file.css && lessc ./www/settings/main.less > ./www/settings/main.css && lessc ./www/slide/slide.less > ./www/slide/slide.css && lessc ./www/whiteboard/whiteboard.less > ./www/whiteboard/whiteboard.css && lessc ./www/poll/poll.less > ./www/poll/poll.css",
-      "template": "cd customize.dist/src && node build.js"
+    "lint": "jshint --config .jshintrc --exclude-path .jshintignore .",
+    "test": "node TestSelenium.js",
+    "style": "lessc ./customize.dist/src/less/cryptpad.less > ./customize.dist/main.css && lessc ./customize.dist/src/less/toolbar.less > ./customize.dist/toolbar.css && lessc ./www/drive/file.less > ./www/drive/file.css && lessc ./www/settings/main.less > ./www/settings/main.css && lessc ./www/slide/slide.less > ./www/slide/slide.css && lessc ./www/whiteboard/whiteboard.less > ./www/whiteboard/whiteboard.css && lessc ./www/poll/poll.less > ./www/poll/poll.css",
+    "template": "cd customize.dist/src && node build.js"
   }
 }
diff --git a/pinneddata.js b/pinneddata.js
new file mode 100644
index 000000000..a24293880
--- /dev/null
+++ b/pinneddata.js
@@ -0,0 +1,101 @@
+/* jshint esversion: 6 */
+const Fs = require('fs');
+const Semaphore = require('saferphore');
+const nThen = require('nthen');
+
+const hashesFromPinFile = (pinFile, fileName) => {
+    var pins = {};
+    pinFile.split('\n').filter((x)=>(x)).map((l) => JSON.parse(l)).forEach((l) => {
+        switch (l[0]) {
+            case 'RESET': {
+                pins = {};
+                //jshint -W086
+                // fallthrough
+            }
+            case 'PIN': {
+                l[1].forEach((x) => { pins[x] = 1; });
+                break;
+            }
+            case 'UNPIN': {
+                l[1].forEach((x) => { delete pins[x]; });
+                break;
+            }
+            default: throw new Error(JSON.stringify(l) + '  ' + fileName);
+        }
+    });
+    return Object.keys(pins);
+};
+
+const sizeForHashes = (hashes, dsFileSizes) => {
+    let sum = 0;
+    hashes.forEach((h) => {
+        const s = dsFileSizes[h];
+        if (typeof(s) !== 'number') {
+            //console.log('missing ' + h + '  ' + typeof(s));
+        } else {
+            sum += s;
+        }
+    });
+    return sum;
+};
+
+const sema = Semaphore.create(20);
+
+let dirList;
+const fileList = [];
+const dsFileSizes = {};
+const out = [];
+
+nThen((waitFor) => {
+    Fs.readdir('./datastore', waitFor((err, list) => {
+        if (err) { throw err; }
+        dirList = list;
+    }));
+}).nThen((waitFor) => {
+    dirList.forEach((f) => {
+        sema.take((returnAfter) => {
+            Fs.readdir('./datastore/' + f, waitFor(returnAfter((err, list2) => {
+                if (err) { throw err; }
+                list2.forEach((ff) => { fileList.push('./datastore/' + f + '/' + ff); });
+            })));
+        });
+    });
+}).nThen((waitFor) => {
+    fileList.forEach((f) => {
+        sema.take((returnAfter) => {
+            Fs.stat(f, waitFor(returnAfter((err, st) => {
+                if (err) { throw err; }
+                dsFileSizes[f.replace(/^.*\/([^\/]*)\.ndjson$/, (all, a) => (a))] = st.size;
+            })));
+        });
+    });
+}).nThen((waitFor) => {
+    Fs.readdir('./pins', waitFor((err, list) => {
+        if (err) { throw err; }
+        dirList = list;
+    }));
+}).nThen((waitFor) => {
+    fileList.splice(0, fileList.length);
+    dirList.forEach((f) => {
+        sema.take((returnAfter) => {
+            Fs.readdir('./pins/' + f, waitFor(returnAfter((err, list2) => {
+                if (err) { throw err; }
+                list2.forEach((ff) => { fileList.push('./pins/' + f + '/' + ff); });
+            })));
+        });
+    });
+}).nThen((waitFor) => {
+    fileList.forEach((f) => {
+        sema.take((returnAfter) => {
+            Fs.readFile(f, waitFor(returnAfter((err, content) => {
+                if (err) { throw err; }
+                const hashes = hashesFromPinFile(content.toString('utf8'), f);
+                const size = sizeForHashes(hashes, dsFileSizes);
+                out.push([f, Math.floor(size / (1024 * 1024))]);
+            })));
+        });
+    });
+}).nThen((waitFor) => {
+    out.sort((a,b) => (a[1] - b[1]));
+    out.forEach((x) => { console.log(x[0] + '  ' + x[1] + ' MB'); });
+});