infra
/
cryptpad
2
1
Fork 0
Code Issues Pull Requests Projects Releases 1 Activity

Merge branch 'staging' into team

pull/1/head
yflory 2019-09-23 11:35:20 +02:00
parent 4381ff8107 33623bdafc
commit 78022fa527
6 changed files with 411 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
customize.dist/pages.js
View File

@ -103,7 +103,7 @@ define([
])*/
])
]),
h('div.cp-version-footer', "CryptPad v3.1.0 (Baiji)")
h('div.cp-version-footer', "CryptPad v3.2.0 (Chilihueque)")
]);
};

5
package.json
View File

@ -1,7 +1,7 @@
{
"name": "cryptpad",
"description": "realtime collaborative visual editor with zero knowlege server",
"version": "3.1.0",
"version": "3.2.0",
"license": "AGPL-3.0+",
"repository": {
"type": "git",
@ -41,6 +41,7 @@
"flow": "./node_modules/.bin/flow",
"test": "node scripts/TestSelenium.js",
"test-rpc": "cd scripts/tests && node test-rpc",
"template": "cd customize.dist/src && for page in ../index.html ../privacy.html ../terms.html ../about.html ../contact.html ../what-is-cryptpad.html ../features.html ../../www/login/index.html ../../www/register/index.html ../../www/user/index.html;do echo $page; cp template.html $page; done;"
"template": "cd customize.dist/src && for page in ../index.html ../privacy.html ../terms.html ../about.html ../contact.html ../what-is-cryptpad.html ../features.html ../../www/login/index.html ../../www/register/index.html ../../www/user/index.html;do echo $page; cp template.html $page; done;",
"evict-inactive": "node scripts/evict-inactive.js"
}
}

32
rpc.js
View File

@ -820,7 +820,21 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) {
}));
}).nThen(function (w) {
// remove the blob
Env.blobStore.remove(blobId, w(function (err) {
if (Env.retainData) {
return void Env.blobStore.archive.blob(blobId, w(function (err) {
Log.info('ARCHIVAL_OWNED_FILE_BY_OWNER_RPC', {
safeKey: safeKey,
blobId: blobId,
status: err? String(err): 'SUCCESS',
});
if (err) {
w.abort();
return void cb(err);
}
}));
}
Env.blobStore.remove.blob(blobId, w(function (err) {
Log.info('DELETION_OWNED_FILE_BY_OWNER_RPC', {
safeKey: safeKey,
blobId: blobId,
@ -833,7 +847,20 @@ var removeOwnedChannel = function (Env, channelId, unsafeKey, cb) {
}));
}).nThen(function () {
// remove the proof
Env.blobStore.removeProof(safeKey, blobId, function (err) {
if (Env.retainData) {
return void Env.blobStore.archive.proof(safeKey, blobId, function (err) {
Log.info("ARCHIVAL_PROOF_REMOVAL_BY_OWNER_RPC", {
safeKey: safeKey,
blobId: blobId,
status: err? String(err): 'SUCCESS',
});
if (err) {
return void cb("E_PROOF_REMOVAL");
}
});
}
Env.blobStore.remove.proof(safeKey, blobId, function (err) {
Log.info("DELETION_PROOF_REMOVAL_BY_OWNER_RPC", {
safeKey: safeKey,
blobId: blobId,
@ -1692,6 +1719,7 @@ RPC.create = function (
BlobStore.create({
blobPath: config.blobPath,
blobStagingPath: config.blobStagingPath,
archivePath: config.archivePath,
getSession: function (safeKey) {
return getSession(Sessions, safeKey);
},

117
scripts/evict-inactive.js
View File

@ -1,6 +1,7 @@
var nThen = require("nthen");
var Store = require("../storage/file");
var BlobStore = require("../storage/blob");
var Pinned = require("./pinned");
var config = require("../lib/load-config");
@ -14,9 +15,16 @@ var inactiveTime = +new Date() - (config.inactiveTime * 24 * 3600 * 1000);
// files which were archived before this date can be considered safe to remove
var retentionTime = +new Date() - (config.archiveRetentionTime * 24 * 3600 * 1000);
var getNewestTime = function (stats) {
return stats[['atime', 'ctime', 'mtime'].reduce(function (a, b) {
return stats[b] > stats[a]? b: a;
})];
};
var store;
var pins;
var Log;
var blobs;
nThen(function (w) {
// load the store which will be used for iterating over channels
// and performing operations like archival and deletion
@ -40,8 +48,17 @@ nThen(function (w) {
Logger.create(config, w(function (_) {
Log = _;
}));
config.getSession = function () {};
BlobStore.create(config, w(function (err, _) {
if (err) {
w.abort();
return console.error(err);
}
blobs = _;
}));
}).nThen(function (w) {
// this block will iterate over archived channels and remove them
// this block will iterate over archived channels and removes them
// if they've been in cold storage for longer than your configured archive time
// if the admin has not set an 'archiveRetentionTime', this block makes no sense
@ -89,6 +106,104 @@ nThen(function (w) {
};
store.listArchivedChannels(handler, w(done));
}).nThen(function (w) {
if (typeof(config.archiveRetentionTime) !== "number") { return; }
var removed = 0;
blobs.list.archived.proofs(function (err, item, next) {
if (err) {
Log.error("EVICT_BLOB_LIST_ARCHIVED_PROOF_ERROR", err);
return void next();
}
if (pins[item.blobId]) { return void next(); }
if (item && getNewestTime(item) > retentionTime) { return void next(); }
blobs.remove.archived.proof(item.safeKey, item.blobId, (function (err) {
if (err) {
Log.error("EVICT_ARCHIVED_BLOB_PROOF_ERROR", item);
return void next();
}
Log.info("EVICT_ARCHIVED_BLOB_PROOF", item);
removed++;
next();
}));
}, w(function () {
Log.info('EVICT_ARCHIVED_BLOB_PROOFS_REMOVED', removed);
}));
}).nThen(function (w) {
if (typeof(config.archiveRetentionTime) !== "number") { return; }
var removed = 0;
blobs.list.archived.blobs(function (err, item, next) {
if (err) {
Log.error("EVICT_BLOB_LIST_ARCHIVED_BLOBS_ERROR", err);
return void next();
}
if (pins[item.blobId]) { return void next(); }
if (item && getNewestTime(item) > retentionTime) { return void next(); }
blobs.remove.archived.blob(item.blobId, function (err) {
if (err) {
Log.error("EVICT_ARCHIVED_BLOB_ERROR", item);
return void next();
}
Log.info("EVICT_ARCHIVED_BLOB", item);
removed++;
next();
});
}, w(function () {
Log.info('EVICT_ARCHIVED_BLOBS_REMOVED', removed);
}));
/* TODO find a reliable metric for determining the activity of blobs...
}).nThen(function (w) {
var blobCount = 0;
var lastHour = 0;
blobs.list.blobs(function (err, item, next) {
blobCount++;
if (err) {
Log.error("EVICT_BLOB_LIST_BLOBS_ERROR", err);
return void next();
}
if (pins[item.blobId]) { return void next(); }
if (item && getNewestTime(item) > retentionTime) { return void next(); }
// TODO determine when to retire blobs
console.log(item);
next();
}, w(function () {
console.log("Listed %s blobs", blobCount);
console.log("Listed %s blobs accessed in the last hour", lastHour);
}));
}).nThen(function (w) {
var proofCount = 0;
blobs.list.proofs(function (err, item, next) {
proofCount++;
if (err) {
next();
return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err);
}
if (pins[item.blobId]) { return void next(); }
if (item && getNewestTime(item) > retentionTime) { return void next(); }
nThen(function (w) {
blobs.size(item.blobId, w(function (err, size) {
if (err) {
w.abort();
next();
return void Log.error("EVICT_BLOB_LIST_PROOFS_ERROR", err);
}
if (size !== 0) {
w.abort();
next();
}
}));
}).nThen(function () {
blobs.remove.proof(item.safeKey, item.blobId, function (err) {
next();
if (err) {
return Log.error("EVICT_BLOB_PROOF_LONELY_ERROR", item);
}
return Log.info("EVICT_BLOB_PROOF_LONELY", item);
});
});
}, function () {
console.log("Listed %s blob proofs", proofCount);
});
*/
}).nThen(function (w) {
var removed = 0;
var channels = 0;

297
storage/blob.js
View File

@ -5,6 +5,7 @@ var Path = require("path");
var BlobStore = module.exports;
var nThen = require("nthen");
var Semaphore = require("saferphore");
var Util = require("../lib/common-util");
var isValidSafeKey = function (safeKey) {
@ -17,34 +18,31 @@ var isValidId = function (id) {
// helpers
// /blob/<safeKeyPrefix>/<safeKey>/<blobPrefix>
var makePathToBlob = function (Env, blobId) {
return Path.join(Env.blobPath, blobId.slice(0, 2));
var prependArchive = function (Env, path) {
return Path.join(Env.archivePath, path);
};
// /blob/<safeKeyPrefix>/<safeKey>/<blobPrefix>/<blobId>
var makeBlobPath = function (Env, blobId) {
return Path.join(makePathToBlob(Env, blobId), blobId);
};
// /blobstate/<safeKeyPrefix>
var makePathToStage = function (Env, safeKey) {
return Path.join(Env.blobStagingPath, safeKey.slice(0, 2));
return Path.join(Env.blobPath, blobId.slice(0, 2), blobId);
};
// /blobstate/<safeKeyPrefix>/<safeKey>
var makeStagePath = function (Env, safeKey) {
return Path.join(makePathToStage(Env, safeKey), safeKey);
};
// /blob/<safeKeyPrefix>/<safeKey>/<blobPrefix>
var makePathToProof = function (Env, safeKey, blobId) {
return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId);
return Path.join(Env.blobStagingPath, safeKey.slice(0, 2), safeKey);
};
// /blob/<safeKeyPrefix>/<safeKey>/<blobPrefix>/<blobId>
var makeProofPath = function (Env, safeKey, blobId) {
return Path.join(makePathToProof(Env, safeKey, blobId), blobId);
return Path.join(Env.blobPath, safeKey.slice(0, 3), safeKey, blobId.slice(0, 2), blobId);
};
var parseProofPath = function (path) {
var parts = path.split('/');
return {
blobId: parts[parts.length -1],
safeKey: parts[parts.length - 3],
};
};
// getUploadSize: used by
@ -76,10 +74,9 @@ var isFile = function (filePath, cb) {
});
};
var makeFileStream = function (dir, full, cb) {
Fse.mkdirp(dir, function (e) {
var makeFileStream = function (full, cb) {
Fse.mkdirp(Path.dirname(full), function (e) {
if (e || !full) { // !full for pleasing flow, it's already checked
//WARN('makeFileStream', e);
return void cb(e ? e.message : 'INTERNAL_ERROR');
}
@ -125,8 +122,10 @@ var upload = function (Env, safeKey, content, cb) {
return cb('E_OVER_LIMIT');
}
var stagePath = makeStagePath(Env, safeKey);
if (!session.blobstage) {
makeFileStream(makePathToStage(Env, safeKey), makeStagePath(Env, safeKey), function (e, stream) {
makeFileStream(stagePath, function (e, stream) {
if (!stream) { return void cb(e); }
var blobstage = session.blobstage = stream;
@ -170,7 +169,7 @@ var upload_complete = function (Env, safeKey, id, cb) {
nThen(function (w) {
// make sure the path to your final location exists
Fse.mkdirp(makePathToBlob(Env, id), function (e) {
Fse.mkdirp(Path.dirname(newPath), function (e) {
if (e) {
w.abort();
return void cb('RENAME_ERR');
@ -237,10 +236,8 @@ var owned_upload_complete = function (Env, safeKey, id, cb) {
return void cb('EINVAL_CONFIG');
}
var ownPath = makePathToProof(Env, safeKey, id);
var filePath = makePathToBlob(Env, id);
var finalPath = makeBlobPath(Env, id);
var finalOwnPath = makeProofPath(Env, safeKey, id);
// the user wants to move it into blob and create a empty file with the same id
@ -249,13 +246,13 @@ var owned_upload_complete = function (Env, safeKey, id, cb) {
nThen(function (w) {
// make the requisite directory structure using Mkdirp
Fse.mkdirp(filePath, w(function (e /*, path */) {
Fse.mkdirp(Path.dirname(finalPath), w(function (e /*, path */) {
if (e) { // does not throw error if the directory already existed
w.abort();
return void cb(e.code);
}
}));
Fse.mkdirp(ownPath, w(function (e /*, path */) {
Fse.mkdirp(Path.dirname(finalOwnPath), w(function (e /*, path */) {
if (e) { // does not throw error if the directory already existed
w.abort();
return void cb(e.code);
@ -318,8 +315,159 @@ var isOwnedBy = function (Env, safeKey, blobId, cb) {
isFile(proofPath, cb);
};
var listFiles = function (Env, handler, cb) {
cb("NOT_IMPLEMENTED");
// archiveBlob
var archiveBlob = function (Env, blobId, cb) {
var blobPath = makeBlobPath(Env, blobId);
var archivePath = prependArchive(Env, blobPath);
Fse.move(blobPath, archivePath, { overwrite: true }, cb);
};
var removeArchivedBlob = function (Env, blobId, cb) {
var archivePath = prependArchive(Env, makeBlobPath(Env, blobId));
Fs.unlink(archivePath, cb);
};
// restoreBlob
var restoreBlob = function (Env, blobId, cb) {
var blobPath = makeBlobPath(Env, blobId);
var archivePath = prependArchive(Env, blobPath);
Fse.move(archivePath, blobPath, cb);
};
// archiveProof
var archiveProof = function (Env, safeKey, blobId, cb) {
var proofPath = makeProofPath(Env, safeKey, blobId);
var archivePath = prependArchive(Env, proofPath);
Fse.move(proofPath, archivePath, { overwrite: true }, cb);
};
var removeArchivedProof = function (Env, safeKey, blobId, cb) {
var archivedPath = prependArchive(Env, makeProofPath(Env, safeKey, blobId));
Fs.unlink(archivedPath, cb);
};
// restoreProof
var restoreProof = function (Env, safeKey, blobId, cb) {
var proofPath = makeProofPath(Env, safeKey, blobId);
var archivePath = prependArchive(Env, proofPath);
Fse.move(archivePath, proofPath, cb);
};
var makeWalker = function (n, handleChild, done) {
if (!n || typeof(n) !== 'number' || n < 2) { n = 2; }
var W;
nThen(function (w) {
// this asynchronous bit defers the completion of this block until
// synchronous execution has completed. This means you must create
// the walker and start using it synchronously or else it will call back
// prematurely
setTimeout(w());
W = w;
}).nThen(function () {
done();
});
// do no more than 20 jobs at a time
var tasks = Semaphore.create(n);
var recurse = function (path) {
tasks.take(function (give) {
var next = give(W());
nThen(function (w) {
// check if the path is a directory...
Fs.stat(path, w(function (err, stats) {
if (err) { return next(); }
if (!stats.isDirectory()) {
w.abort();
return void handleChild(void 0, path, next);
}
// fall through
}));
}).nThen(function () {
// handle directories
Fs.readdir(path, function (err, dir) {
if (err) { return next(); }
// everything is fine and it's a directory...
dir.forEach(function (d) {
recurse(Path.join(path, d));
});
next();
});
});
});
};
return recurse;
};
var listProofs = function (root, handler, cb) {
Fs.readdir(root, function (err, dir) {
if (err) { return void cb(err); }
var walk = makeWalker(20, function (err, path, next) {
// path is the path to a child node on the filesystem
// next handles the next job in a queue
// iterate over proofs
// check for presence of corresponding files
Fs.stat(path, function (err, stats) {
if (err) {
return void handler(err, void 0, next);
}
var parsed = parseProofPath(path);
handler(void 0, {
path: path,
blobId: parsed.blobId,
safeKey: parsed.safeKey,
atime: stats.atime,
ctime: stats.ctime,
mtime: stats.mtime,
}, next);
});
}, function () {
// called when there are no more directories or children to process
cb();
});
dir.forEach(function (d) {
// ignore directories that aren't 3 characters long...
if (d.length !== 3) { return; }
walk(Path.join(root, d));
});
});
};
var listBlobs = function (root, handler, cb) {
// iterate over files
Fs.readdir(root, function (err, dir) {
if (err) { return void cb(err); }
var walk = makeWalker(20, function (err, path, next) {
Fs.stat(path, function (err, stats) {
if (err) {
return void handler(err, void 0, next);
}
handler(void 0, {
blobId: Path.basename(path),
atime: stats.atime,
ctime: stats.ctime,
mtime: stats.mtime,
}, next);
});
}, function () {
cb();
});
dir.forEach(function (d) {
if (d.length !== 2) { return; }
walk(Path.join(root, d));
});
});
};
BlobStore.create = function (config, _cb) {
@ -331,6 +479,7 @@ BlobStore.create = function (config, _cb) {
var Env = {
blobPath: config.blobPath || './blob',
blobStagingPath: config.blobStagingPath || './blobstage',
archivePath: config.archivePath || './data/archive',
getSession: config.getSession,
};
@ -342,8 +491,12 @@ BlobStore.create = function (config, _cb) {
Fse.mkdirp(Env.blobStagingPath, w(function (e) {
if (e) { CB(e); }
}));
Fse.mkdirp(Path.join(Env.archivePath, Env.blobPath), w(function (e) {
if (e) { CB(e); }
}));
}).nThen(function () {
cb(void 0, {
var methods = {
isFileId: isValidId,
status: function (safeKey, _cb) {
// TODO check if the final destination is a file
@ -372,17 +525,59 @@ BlobStore.create = function (config, _cb) {
isOwnedBy(Env, safeKey, blobId, cb);
},
remove: function (blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
remove(Env, blobId, cb);
remove: {
blob: function (blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
remove(Env, blobId, cb);
},
proof: function (safeKey, blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
removeProof(Env, safeKey, blobId, cb);
},
archived: {
blob: function (blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
removeArchivedBlob(Env, blobId, cb);
},
proof: function (safeKey, blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
removeArchivedProof(Env, safeKey, blobId, cb);
},
},
},
removeProof: function (safeKey, blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
removeProof(Env, safeKey, blobId, cb);
archive: {
blob: function (blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
archiveBlob(Env, blobId, cb);
},
proof: function (safeKey, blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
archiveProof(Env, safeKey, blobId, cb);
},
},
restore: {
blob: function (blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
restoreBlob(Env, blobId, cb);
},
proof: function (safeKey, blobId, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
if (!isValidSafeKey(safeKey)) { return void cb('INVALID_SAFEKEY'); }
if (!isValidId(blobId)) { return void cb("INVALID_ID"); }
restoreProof(Env, safeKey, blobId, cb);
},
},
complete: function (safeKey, id, _cb) {
@ -403,11 +598,29 @@ BlobStore.create = function (config, _cb) {
getUploadSize(Env, id, cb);
},
list: function (handler, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
listFiles(Env, handler, cb);
list: {
blobs: function (handler, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
listBlobs(Env.blobPath, handler, cb);
},
proofs: function (handler, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
listProofs(Env.blobPath, handler, cb);
},
archived: {
proofs: function (handler, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
listProofs(prependArchive(Env, Env.blobPath), handler, cb);
},
blobs: function (handler, _cb) {
var cb = Util.once(Util.mkAsync(_cb));
listBlobs(prependArchive(Env, Env.blobPath), handler, cb);
},
}
},
});
};
cb(void 0, methods);
});
};

6
www/common/outer/roster.js
View File

@ -360,6 +360,7 @@ var factory = function (Util, Hash, CPNetflux, Sortify, nThen, Crypto) {
internal: {
initialized: false,
sinceLastCheckpoint: 0,
lastCheckpointHash: -1
},
};
var roster = {};
@ -395,6 +396,10 @@ var factory = function (Util, Hash, CPNetflux, Sortify, nThen, Crypto) {
return Util.clone(ref.state);
};
roster.getLastCheckpointHash = function () {
return ref.internal.lastCheckpointHash || -1;
};
var clearPendingCheckpoints = function () {
// clear any pending checkpoints you might have...
if (ref.internal.pendingCheckpointId) {
@ -481,6 +486,7 @@ var factory = function (Util, Hash, CPNetflux, Sortify, nThen, Crypto) {
events.checkpoint.fire(hash);
// reset the counter for messages since the last checkpoint
ref.internal.sinceLastCheckpoint = 0;
ref.internal.lastCheckpointHash = hash;
} else if (changed) {
events.change.fire();
}