diff --git a/customize.dist/pages.js b/customize.dist/pages.js index bd373b04c..43702f7cd 100644 --- a/customize.dist/pages.js +++ b/customize.dist/pages.js @@ -103,7 +103,7 @@ define([ ])*/ ]) ]), - h('div.cp-version-footer', "CryptPad v3.1.0 (Baiji)") + h('div.cp-version-footer', "CryptPad v3.2.0 (Chilihueque)") ]); }; diff --git a/package.json b/package.json index fdbbc008a..d47aa5a2e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "cryptpad", "description": "realtime collaborative visual editor with zero knowlege server", - "version": "3.1.0", + "version": "3.2.0", "license": "AGPL-3.0+", "repository": { "type": "git", @@ -41,6 +41,7 @@ "flow": "./node_modules/.bin/flow", "test": "node scripts/TestSelenium.js", "test-rpc": "cd scripts/tests && node test-rpc", - "template": "cd customize.dist/src && for page in ../index.html ../privacy.html ../terms.html ../about.html ../contact.html ../what-is-cryptpad.html ../features.html ../../www/login/index.html ../../www/register/index.html ../../www/user/index.html;do echo $page; cp template.html $page; done;" + "template": "cd customize.dist/src && for page in ../index.html ../privacy.html ../terms.html ../about.html ../contact.html ../what-is-cryptpad.html ../features.html ../../www/login/index.html ../../www/register/index.html ../../www/user/index.html;do echo $page; cp template.html $page; done;", + "evict-inactive": "node scripts/evict-inactive.js" } } diff --git a/rpc.js b/rpc.js index 62a8690a2..f7d2d2fcb 100644 --- a/rpc.js +++ b/rpc.js @@ -820,7 +820,21 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { })); }).nThen(function (w) { // remove the blob - Env.blobStore.remove(blobId, w(function (err) { + + if (Env.retainData) { + return void Env.blobStore.archive.blob(blobId, w(function (err) { + Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', { + safeKey: safeKey, + blobId: blobId, + status: err? String(err): 'SUCCESS', + }); + if (err) { + w.abort(); + return void cb(err); + } + })); + } + Env.blobStore.remove.blob(blobId, w(function (err) { Log.info('DELETION_OWNED_FILE_BY_OWNER_RPC', { safeKey: safeKey, blobId: blobId, @@ -833,7 +847,20 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) { })); }).nThen(function () { // remove the proof - Env.blobStore.removeProof(safeKey, blobId, function (err) { + if (Env.retainData) { + return void Env.blobStore.archive.proof(safeKey, blobId, function (err) { + Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", { + safeKey: safeKey, + blobId: blobId, + status: err? String(err): 'SUCCESS', + }); + if (err) { + return void cb("E_PROOF_REMOVAL"); + } + }); + } + + Env.blobStore.remove.proof(safeKey, blobId, function (err) { Log.info("DELETION_PROOF_REMOVAL_BY_OWNER_RPC", { safeKey: safeKey, blobId: blobId, @@ -1692,6 +1719,7 @@ RPC.create = function ( BlobStore.create({ blobPath: config.blobPath, blobStagingPath: config.blobStagingPath, + archivePath: config.archivePath, getSession: function (safeKey) { return getSession(Sessions, safeKey); }, diff --git a/scripts/evict-inactive.js b/scripts/evict-inactive.js index ff9a3c343..ebb65d637 100644 --- a/scripts/evict-inactive.js +++ b/scripts/evict-inactive.js @@ -1,6 +1,7 @@ var nThen = require("nthen"); var Store = require("../storage/file"); +var BlobStore = require("../storage/blob"); var Pinned = require("./pinned"); var config = require("../lib/load-config"); @@ -14,9 +15,16 @@ var inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000); // files which were archived before this date can be considered safe to remove var retentionTime = +new Date() - (config.archiveRetentionTime * 24 * 3600 * 1000); +var getNewestTime = function (stats) { + return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) { + return stats[b] > stats[a]? b: a; + })]; +}; + var store; var pins; var Log; +var blobs; nThen(function (w) { // load the store which will be used for iterating over channels // and performing operations like archival and deletion @@ -40,8 +48,17 @@ nThen(function (w) { Logger.create(config, w(function (_) { Log = _; })); + + config.getSession = function () {}; + BlobStore.create(config, w(function (err, _) { + if (err) { + w.abort(); + return console.error(err); + } + blobs = _; + })); }).nThen(function (w) { - // this block will iterate over archived channels and remove them + // this block will iterate over archived channels and removes them // if they've been in cold storage for longer than your configured archive time // if the admin has not set an 'archiveRetentionTime', this block makes no sense @@ -89,6 +106,104 @@ nThen(function (w) { }; store.listArchivedChannels(handler, w(done)); +}).nThen(function (w) { + if (typeof(config.archiveRetentionTime) !== "number") { return; } + var removed = 0; + blobs.list.archived.proofs(function (err, item, next) { + if (err) { + Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err); + return void next(); + } + if (pins[item.blobId]) { return void next(); } + if (item && getNewestTime(item) > retentionTime) { return void next(); } + blobs.remove.archived.proof(item.safeKey, item.blobId, (function (err) { + if (err) { + Log.error("EVICT_ARCHIVED_BLOB_PROOF_ERROR", item); + return void next(); + } + Log.info("EVICT_ARCHIVED_BLOB_PROOF", item); + removed++; + next(); + })); + }, w(function () { + Log.info('EVICT_ARCHIVED_BLOB_PROOFS_REMOVED', removed); + })); +}).nThen(function (w) { + if (typeof(config.archiveRetentionTime) !== "number") { return; } + var removed = 0; + blobs.list.archived.blobs(function (err, item, next) { + if (err) { + Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err); + return void next(); + } + if (pins[item.blobId]) { return void next(); } + if (item && getNewestTime(item) > retentionTime) { return void next(); } + blobs.remove.archived.blob(item.blobId, function (err) { + if (err) { + Log.error("EVICT_ARCHIVED_BLOB_ERROR", item); + return void next(); + } + Log.info("EVICT_ARCHIVED_BLOB", item); + removed++; + next(); + }); + }, w(function () { + Log.info('EVICT_ARCHIVED_BLOBS_REMOVED', removed); + })); +/* TODO find a reliable metric for determining the activity of blobs... +}).nThen(function (w) { + var blobCount = 0; + var lastHour = 0; + blobs.list.blobs(function (err, item, next) { + blobCount++; + if (err) { + Log.error("EVICT_BLOB_LIST_BLOBS_ERROR", err); + return void next(); + } + if (pins[item.blobId]) { return void next(); } + if (item && getNewestTime(item) > retentionTime) { return void next(); } + // TODO determine when to retire blobs + console.log(item); + next(); + }, w(function () { + console.log("Listed %s blobs", blobCount); + console.log("Listed %s blobs accessed in the last hour", lastHour); + })); +}).nThen(function (w) { + var proofCount = 0; + blobs.list.proofs(function (err, item, next) { + proofCount++; + if (err) { + next(); + return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err); + } + if (pins[item.blobId]) { return void next(); } + if (item && getNewestTime(item) > retentionTime) { return void next(); } + nThen(function (w) { + blobs.size(item.blobId, w(function (err, size) { + if (err) { + w.abort(); + next(); + return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err); + } + if (size !== 0) { + w.abort(); + next(); + } + })); + }).nThen(function () { + blobs.remove.proof(item.safeKey, item.blobId, function (err) { + next(); + if (err) { + return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item); + } + return Log.info("EVICT_BLOB_PROOF_LONELY", item); + }); + }); + }, function () { + console.log("Listed %s blob proofs", proofCount); + }); +*/ }).nThen(function (w) { var removed = 0; var channels = 0; diff --git a/storage/blob.js b/storage/blob.js index aa0f43528..6b3e891dd 100644 --- a/storage/blob.js +++ b/storage/blob.js @@ -5,6 +5,7 @@ var Path = require("path"); var BlobStore = module.exports; var nThen = require("nthen"); +var Semaphore = require("saferphore"); var Util = require("../lib/common-util"); var isValidSafeKey = function (safeKey) { @@ -17,34 +18,31 @@ var isValidId = function (id) { // helpers -// /blob/// -var makePathToBlob = function (Env, blobId) { - return Path.join(Env.blobPath, blobId.slice(0, 2)); +var prependArchive = function (Env, path) { + return Path.join(Env.archivePath, path); }; // /blob//// var makeBlobPath = function (Env, blobId) { - return Path.join(makePathToBlob(Env, blobId), blobId); -}; - -// /blobstate/ -var makePathToStage = function (Env, safeKey) { - return Path.join(Env.blobStagingPath, safeKey.slice(0, 2)); + return Path.join(Env.blobPath, blobId.slice(0, 2), blobId); }; // /blobstate// var makeStagePath = function (Env, safeKey) { - return Path.join(makePathToStage(Env, safeKey), safeKey); -}; - -// /blob/// -var makePathToProof = function (Env, safeKey, blobId) { - return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId); + return Path.join(Env.blobStagingPath, safeKey.slice(0, 2), safeKey); }; // /blob//// var makeProofPath = function (Env, safeKey, blobId) { - return Path.join(makePathToProof(Env, safeKey, blobId), blobId); + return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId); +}; + +var parseProofPath = function (path) { + var parts = path.split('/'); + return { + blobId: parts[parts.length -1], + safeKey: parts[parts.length - 3], + }; }; // getUploadSize: used by @@ -76,10 +74,9 @@ var isFile = function (filePath, cb) { }); }; -var makeFileStream = function (dir, full, cb) { - Fse.mkdirp(dir, function (e) { +var makeFileStream = function (full, cb) { + Fse.mkdirp(Path.dirname(full), function (e) { if (e || !full) { // !full for pleasing flow, it's already checked - //WARN('makeFileStream', e); return void cb(e ? e.message : 'INTERNAL_ERROR'); } @@ -125,8 +122,10 @@ var upload = function (Env, safeKey, content, cb) { return cb('E_OVER_LIMIT'); } + var stagePath = makeStagePath(Env, safeKey); + if (!session.blobstage) { - makeFileStream(makePathToStage(Env, safeKey), makeStagePath(Env, safeKey), function (e, stream) { + makeFileStream(stagePath, function (e, stream) { if (!stream) { return void cb(e); } var blobstage = session.blobstage = stream; @@ -170,7 +169,7 @@ var upload_complete = function (Env, safeKey, id, cb) { nThen(function (w) { // make sure the path to your final location exists - Fse.mkdirp(makePathToBlob(Env, id), function (e) { + Fse.mkdirp(Path.dirname(newPath), function (e) { if (e) { w.abort(); return void cb('RENAME_ERR'); @@ -237,10 +236,8 @@ var owned_upload_complete = function (Env, safeKey, id, cb) { return void cb('EINVAL_CONFIG'); } - var ownPath = makePathToProof(Env, safeKey, id); - var filePath = makePathToBlob(Env, id); - var finalPath = makeBlobPath(Env, id); + var finalOwnPath = makeProofPath(Env, safeKey, id); // the user wants to move it into blob and create a empty file with the same id @@ -249,13 +246,13 @@ var owned_upload_complete = function (Env, safeKey, id, cb) { nThen(function (w) { // make the requisite directory structure using Mkdirp - Fse.mkdirp(filePath, w(function (e /*, path */) { + Fse.mkdirp(Path.dirname(finalPath), w(function (e /*, path */) { if (e) { // does not throw error if the directory already existed w.abort(); return void cb(e.code); } })); - Fse.mkdirp(ownPath, w(function (e /*, path */) { + Fse.mkdirp(Path.dirname(finalOwnPath), w(function (e /*, path */) { if (e) { // does not throw error if the directory already existed w.abort(); return void cb(e.code); @@ -318,8 +315,159 @@ var isOwnedBy = function (Env, safeKey, blobId, cb) { isFile(proofPath, cb); }; -var listFiles = function (Env, handler, cb) { - cb("NOT_IMPLEMENTED"); + +// archiveBlob +var archiveBlob = function (Env, blobId, cb) { + var blobPath = makeBlobPath(Env, blobId); + var archivePath = prependArchive(Env, blobPath); + Fse.move(blobPath, archivePath, { overwrite: true }, cb); +}; + +var removeArchivedBlob = function (Env, blobId, cb) { + var archivePath = prependArchive(Env, makeBlobPath(Env, blobId)); + Fs.unlink(archivePath, cb); +}; + +// restoreBlob +var restoreBlob = function (Env, blobId, cb) { + var blobPath = makeBlobPath(Env, blobId); + var archivePath = prependArchive(Env, blobPath); + Fse.move(archivePath, blobPath, cb); +}; + +// archiveProof +var archiveProof = function (Env, safeKey, blobId, cb) { + var proofPath = makeProofPath(Env, safeKey, blobId); + var archivePath = prependArchive(Env, proofPath); + Fse.move(proofPath, archivePath, { overwrite: true }, cb); +}; + +var removeArchivedProof = function (Env, safeKey, blobId, cb) { + var archivedPath = prependArchive(Env, makeProofPath(Env, safeKey, blobId)); + Fs.unlink(archivedPath, cb); +}; + +// restoreProof +var restoreProof = function (Env, safeKey, blobId, cb) { + var proofPath = makeProofPath(Env, safeKey, blobId); + var archivePath = prependArchive(Env, proofPath); + Fse.move(archivePath, proofPath, cb); +}; + +var makeWalker = function (n, handleChild, done) { + if (!n || typeof(n) !== 'number' || n < 2) { n = 2; } + + var W; + nThen(function (w) { + // this asynchronous bit defers the completion of this block until + // synchronous execution has completed. This means you must create + // the walker and start using it synchronously or else it will call back + // prematurely + setTimeout(w()); + W = w; + }).nThen(function () { + done(); + }); + + // do no more than 20 jobs at a time + var tasks = Semaphore.create(n); + + var recurse = function (path) { + tasks.take(function (give) { + var next = give(W()); + + nThen(function (w) { + // check if the path is a directory... + Fs.stat(path, w(function (err, stats) { + if (err) { return next(); } + if (!stats.isDirectory()) { + w.abort(); + return void handleChild(void 0, path, next); + } + // fall through + })); + }).nThen(function () { + // handle directories + Fs.readdir(path, function (err, dir) { + if (err) { return next(); } + // everything is fine and it's a directory... + dir.forEach(function (d) { + recurse(Path.join(path, d)); + }); + next(); + }); + }); + }); + }; + + return recurse; +}; + +var listProofs = function (root, handler, cb) { + Fs.readdir(root, function (err, dir) { + if (err) { return void cb(err); } + + var walk = makeWalker(20, function (err, path, next) { + // path is the path to a child node on the filesystem + + // next handles the next job in a queue + + // iterate over proofs + // check for presence of corresponding files + Fs.stat(path, function (err, stats) { + if (err) { + return void handler(err, void 0, next); + } + + var parsed = parseProofPath(path); + handler(void 0, { + path: path, + blobId: parsed.blobId, + safeKey: parsed.safeKey, + atime: stats.atime, + ctime: stats.ctime, + mtime: stats.mtime, + }, next); + }); + }, function () { + // called when there are no more directories or children to process + cb(); + }); + + dir.forEach(function (d) { + // ignore directories that aren't 3 characters long... + if (d.length !== 3) { return; } + walk(Path.join(root, d)); + }); + }); +}; + +var listBlobs = function (root, handler, cb) { + // iterate over files + Fs.readdir(root, function (err, dir) { + if (err) { return void cb(err); } + var walk = makeWalker(20, function (err, path, next) { + Fs.stat(path, function (err, stats) { + if (err) { + return void handler(err, void 0, next); + } + + handler(void 0, { + blobId: Path.basename(path), + atime: stats.atime, + ctime: stats.ctime, + mtime: stats.mtime, + }, next); + }); + }, function () { + cb(); + }); + + dir.forEach(function (d) { + if (d.length !== 2) { return; } + walk(Path.join(root, d)); + }); + }); }; BlobStore.create = function (config, _cb) { @@ -331,6 +479,7 @@ BlobStore.create = function (config, _cb) { var Env = { blobPath: config.blobPath || './blob', blobStagingPath: config.blobStagingPath || './blobstage', + archivePath: config.archivePath || './data/archive', getSession: config.getSession, }; @@ -342,8 +491,12 @@ BlobStore.create = function (config, _cb) { Fse.mkdirp(Env.blobStagingPath, w(function (e) { if (e) { CB(e); } })); + + Fse.mkdirp(Path.join(Env.archivePath, Env.blobPath), w(function (e) { + if (e) { CB(e); } + })); }).nThen(function () { - cb(void 0, { + var methods = { isFileId: isValidId, status: function (safeKey, _cb) { // TODO check if the final destination is a file @@ -372,17 +525,59 @@ BlobStore.create = function (config, _cb) { isOwnedBy(Env, safeKey, blobId, cb); }, - remove: function (blobId, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - if (!isValidId(blobId)) { return void cb("INVALID_ID"); } - remove(Env, blobId, cb); + remove: { + blob: function (blobId, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidId(blobId)) { return void cb("INVALID_ID"); } + remove(Env, blobId, cb); + }, + proof: function (safeKey, blobId, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); } + if (!isValidId(blobId)) { return void cb("INVALID_ID"); } + removeProof(Env, safeKey, blobId, cb); + }, + archived: { + blob: function (blobId, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidId(blobId)) { return void cb("INVALID_ID"); } + removeArchivedBlob(Env, blobId, cb); + }, + proof: function (safeKey, blobId, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); } + if (!isValidId(blobId)) { return void cb("INVALID_ID"); } + removeArchivedProof(Env, safeKey, blobId, cb); + }, + }, }, - removeProof: function (safeKey, blobId, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); } - if (!isValidId(blobId)) { return void cb("INVALID_ID"); } - removeProof(Env, safeKey, blobId, cb); + archive: { + blob: function (blobId, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidId(blobId)) { return void cb("INVALID_ID"); } + archiveBlob(Env, blobId, cb); + }, + proof: function (safeKey, blobId, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); } + if (!isValidId(blobId)) { return void cb("INVALID_ID"); } + archiveProof(Env, safeKey, blobId, cb); + }, + }, + + restore: { + blob: function (blobId, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidId(blobId)) { return void cb("INVALID_ID"); } + restoreBlob(Env, blobId, cb); + }, + proof: function (safeKey, blobId, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); } + if (!isValidId(blobId)) { return void cb("INVALID_ID"); } + restoreProof(Env, safeKey, blobId, cb); + }, }, complete: function (safeKey, id, _cb) { @@ -403,11 +598,29 @@ BlobStore.create = function (config, _cb) { getUploadSize(Env, id, cb); }, - list: function (handler, _cb) { - var cb = Util.once(Util.mkAsync(_cb)); - listFiles(Env, handler, cb); + list: { + blobs: function (handler, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + listBlobs(Env.blobPath, handler, cb); + }, + proofs: function (handler, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + listProofs(Env.blobPath, handler, cb); + }, + archived: { + proofs: function (handler, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + listProofs(prependArchive(Env, Env.blobPath), handler, cb); + }, + blobs: function (handler, _cb) { + var cb = Util.once(Util.mkAsync(_cb)); + listBlobs(prependArchive(Env, Env.blobPath), handler, cb); + }, + } }, - }); + }; + + cb(void 0, methods); }); }; diff --git a/www/common/outer/roster.js b/www/common/outer/roster.js index b9e2159f9..0cc37d35c 100644 --- a/www/common/outer/roster.js +++ b/www/common/outer/roster.js @@ -360,6 +360,7 @@ var factory = function (Util, Hash, CPNetflux, Sortify, nThen, Crypto) { internal: { initialized: false, sinceLastCheckpoint: 0, + lastCheckpointHash: -1 }, }; var roster = {}; @@ -395,6 +396,10 @@ var factory = function (Util, Hash, CPNetflux, Sortify, nThen, Crypto) { return Util.clone(ref.state); }; + roster.getLastCheckpointHash = function () { + return ref.internal.lastCheckpointHash || -1; + }; + var clearPendingCheckpoints = function () { // clear any pending checkpoints you might have... if (ref.internal.pendingCheckpointId) { @@ -481,6 +486,7 @@ var factory = function (Util, Hash, CPNetflux, Sortify, nThen, Crypto) { events.checkpoint.fire(hash); // reset the counter for messages since the last checkpoint ref.internal.sinceLastCheckpoint = 0; + ref.internal.lastCheckpointHash = hash; } else if (changed) { events.change.fire(); }