Migration: add a tool to migrate to etesync 2.0.

master
Tom Hacohen 4 years ago
parent 5a5c777619
commit f579f3fcb0

@ -9,6 +9,7 @@
"@material-ui/lab": "^4.0.0-alpha.47", "@material-ui/lab": "^4.0.0-alpha.47",
"@material-ui/pickers": "^3.2.10", "@material-ui/pickers": "^3.2.10",
"@material-ui/styles": "^4.6.0", "@material-ui/styles": "^4.6.0",
"etebase": "^0.11.0",
"etesync": "^0.3.1", "etesync": "^0.3.1",
"fuse.js": "^5.0.9-beta", "fuse.js": "^5.0.9-beta",
"ical.js": "^1.4.0", "ical.js": "^1.4.0",

@ -89,6 +89,8 @@ export const routeResolver = new RouteResolver({
}, },
debug: { debug: {
}, },
"migrate-v2": {
},
}); });
interface AppBarPropsType { interface AppBarPropsType {
@ -143,7 +145,7 @@ function App(props: PropsType) {
function autoRefresh() { function autoRefresh() {
if (navigator.onLine && props.credentials && if (navigator.onLine && props.credentials &&
!(window.location.pathname.match(/.*\/(new|edit|duplicate)$/))) { !(window.location.pathname.match(/.*\/(new|edit|duplicate)$/) || window.location.pathname.startsWith("/migrate-v2"))) {
refresh(); refresh();
} }
} }

@ -0,0 +1,380 @@
// SPDX-FileCopyrightText: © 2017 EteSync Authors
// SPDX-License-Identifier: AGPL-3.0-only
import * as React from "react";
import * as EteSync from "etesync";
import * as Etebase from "etebase";
import Button from "@material-ui/core/Button";
import TextField from "@material-ui/core/TextField";
import Switch from "@material-ui/core/Switch";
import { Map as ImmutableMap } from "immutable";
import ContactsIcon from "@material-ui/icons/Contacts";
import CalendarTodayIcon from "@material-ui/icons/CalendarToday";
import FormatListBulletedIcon from "@material-ui/icons/FormatListBulleted";
import Container from "./widgets/Container";
import { useSelector } from "react-redux";
import { StoreState, CredentialsData, UserInfoData } from "./store";
import AppBarOverride from "./widgets/AppBarOverride";
import ColorBox from "./widgets/ColorBox";
import { List, ListItem } from "./widgets/List";
import { colorIntToHtml } from "./journal-processors";
import { Checkbox, FormGroup, FormControlLabel, CircularProgress } from "@material-ui/core";
import Alert from "@material-ui/lab/Alert";
import { arrayToChunkIterator } from "./helpers";
import { ContactType, EventType, TaskType, PimType } from "./pim-types";
interface PropsType {
etesync: CredentialsData;
userInfo: UserInfoData;
}
interface FormErrors {
errorEmail?: string;
errorPassword?: string;
errorServer?: string;
errorGeneral?: string;
}
export default function MigrateV2(props: PropsType) {
const [wantedJournals, setWantedJournals] = React.useState<ImmutableMap<string, EteSync.Journal>>(ImmutableMap({}));
const [username, setUsername] = React.useState("");
const [password, setPassword] = React.useState("");
const [server, setServer] = React.useState("");
const [showAdvanced, setShowAdvanced] = React.useState(false);
const [loading, setLoading] = React.useState(false);
const [progress, setProgress] = React.useState("");
const [errors, setErrors] = React.useState<FormErrors>({});
const journals = useSelector((state: StoreState) => state.cache.journals!);
const journalEntries = useSelector((state: StoreState) => state.cache.entries);
const derived = props.etesync.encryptionKey;
const decryptedJournals = React.useMemo(() => {
return journals.map((journal) => {
const userInfo = props.userInfo;
const keyPair = userInfo.getKeyPair(userInfo.getCryptoManager(derived));
const cryptoManager = journal.getCryptoManager(derived, keyPair);
const info = journal.getInfo(cryptoManager);
return { journal, info };
});
}, [props.userInfo, journals, derived]);
const journalClicked = React.useCallback((journal: EteSync.Journal) => {
if (wantedJournals.has(journal.uid)) {
setWantedJournals(wantedJournals.remove(journal.uid));
} else {
setWantedJournals(wantedJournals.set(journal.uid, journal));
}
}, [wantedJournals]);
const journalMap = React.useMemo(() => {
return decryptedJournals.reduce(
(ret, { journal, info }) => {
let colorBox: React.ReactElement | undefined;
switch (info.type) {
case "CALENDAR":
case "TASKS":
colorBox = (
<ColorBox size={24} color={colorIntToHtml(info.color)} />
);
break;
}
ret[info.type] = ret[info.type] || [];
ret[info.type].push(
<ListItem key={journal.uid} rightIcon={colorBox} insetChildren
onClick={() => journalClicked(journal)}>
<Checkbox
checked={wantedJournals.has(journal.uid)}
/>
{info.displayName}
</ListItem>
);
return ret;
},
{
CALENDAR: [],
ADDRESS_BOOK: [],
TASKS: [],
});
}, [decryptedJournals, journalClicked]);
const styles = {
form: {
},
forgotPassword: {
paddingTop: 20,
},
textField: {
marginTop: 20,
},
submit: {
marginTop: 40,
},
};
function handleInputChange(func: (value: string) => void) {
return (event: React.ChangeEvent<any>) => {
func(event.target.value);
};
}
async function onSubmit() {
setLoading(true);
setProgress("");
try {
const errors: FormErrors = {};
const fieldRequired = "This field is required!";
if (!username) {
errors.errorEmail = fieldRequired;
}
if (!password) {
errors.errorPassword = fieldRequired;
}
if (showAdvanced && !server.startsWith("http")) {
errors.errorServer = "Server URI must start with http/https://";
}
if (Object.keys(errors).length) {
setErrors(errors);
return;
} else {
setErrors({});
}
const serverUrl = (showAdvanced) ? server : undefined;
let malformed = 0;
setProgress("Logging into EteSync 2.0 account");
const etebase = await Etebase.Account.login(username, password, serverUrl);
const colMgr = etebase.getCollectionManager();
const { etesync, userInfo } = props;
const derived = etesync.encryptionKey;
const userInfoCryptoManager = userInfo.getCryptoManager(etesync.encryptionKey);
const keyPair = userInfo.getKeyPair(userInfoCryptoManager);
const now = (new Date()).getTime();
let i = 1;
for (const journal of wantedJournals.values()) {
setProgress(`Migrating collection ${i}/${wantedJournals.size}`);
console.log("Migrating ", journal.uid);
const cryptoManager = journal.getCryptoManager(derived, keyPair);
const info = journal.getInfo(cryptoManager);
const entries = journalEntries.get(journal.uid)!;
let colType;
let parseFunc: (content: string) => PimType;
switch (info.type) {
case "ADDRESS_BOOK": {
colType = "etebase.vcard";
parseFunc = ContactType.parse;
break;
}
case "CALENDAR": {
colType = "etebase.vevent";
parseFunc = EventType.parse;
break;
}
case "TASKS": {
colType = "etebase.vtodo";
parseFunc = TaskType.parse;
break;
}
default: {
continue;
}
}
const meta: Etebase.CollectionMetadata = {
type: colType,
name: info.displayName,
description: info.description,
color: (info.color !== undefined) ? colorIntToHtml(info.color) : undefined,
};
const collection = await colMgr.create(meta, "");
await colMgr.upload(collection);
const itemMgr = colMgr.getItemManager(collection);
const CHUNK_SIZE = 20;
const items = new Map<string, Etebase.Item>();
let done = 0;
let prevUid: string | null = null;
for (const chunk of arrayToChunkIterator(entries.toArray(), CHUNK_SIZE)) {
setProgress(`Migrating collection ${i}/${wantedJournals.size}\nMigrated entries: ${done}/${entries.size}`);
const chunkItems = [];
for (const entry of chunk) {
console.log("Migrating entry ", entry.uid);
done++;
const syncEntry = entry.getSyncEntry(cryptoManager, prevUid);
prevUid = entry.uid;
const pimItem = parseFunc(syncEntry.content);
const uid = pimItem.uid;
// When we can't set mtime, set to the item's position in the change log so we at least maintain EteSync 1.0 ordering.
const mtime = (pimItem.lastModified?.toJSDate())?.getTime() ?? now + done;
if (!uid) {
malformed++;
continue;
}
let item = items.get(uid);
if (item) {
// Existing item
item = item._clone(); // We are cloning so we can push multiple revisions at once
await item.setContent(syncEntry.content);
const meta = await item.getMeta();
meta.mtime = mtime;
await item.setMeta(meta);
} else {
// New
const meta: Etebase.ItemMetadata = {
mtime,
name: uid,
};
item = await itemMgr.create(meta, syncEntry.content);
items.set(uid, item);
}
if (syncEntry.action === EteSync.SyncEntryAction.Delete) {
item.delete(true);
}
chunkItems.push(item);
}
await itemMgr.batch(chunkItems);
}
i++;
}
await etebase.logout();
if (malformed > 0) {
setProgress(`Done\nIgnored ${malformed} entries (probably safe to ignore)`);
} else {
setProgress("Done");
}
} catch (e) {
if (e instanceof Etebase.UnauthorizedError) {
errors.errorGeneral = "Wrong username or password";
} else {
errors.errorGeneral = e.toString();
}
setErrors(errors);
} finally {
setLoading(false);
}
}
let advancedSettings = null;
if (showAdvanced) {
advancedSettings = (
<React.Fragment>
<TextField
type="url"
style={styles.textField}
error={!!errors.errorServer}
helperText={errors.errorServer}
label="Server"
name="server"
value={server}
onChange={handleInputChange(setServer)}
/>
<br />
</React.Fragment>
);
}
return (
<Container>
<AppBarOverride title="Migrate to EteSync 2.0" />
<p>
This tool will help you migrate your data to EteSync 2.0.
</p>
<p>
The migration doesn't delete any data. It only copies your data over to the new EteSync 2.0 server. This means that there is no risk of data-loss in the migration.
</p>
<p>
Please select the collections you would like to migrate, and then enter your EteSync 2.0 credentials and click migrate.
</p>
<List>
<ListItem
primaryText="Address Books"
leftIcon={<ContactsIcon />}
nestedItems={journalMap.ADDRESS_BOOK}
/>
<ListItem
primaryText="Calendars"
leftIcon={<CalendarTodayIcon />}
nestedItems={journalMap.CALENDAR}
/>
<ListItem
primaryText="Tasks"
leftIcon={<FormatListBulletedIcon />}
nestedItems={journalMap.TASKS}
/>
</List>
<h3>EteSync 2.0 credentials</h3>
<TextField
type="text"
style={styles.textField}
error={!!errors.errorEmail}
helperText={errors.errorEmail}
label="Username"
name="username"
value={username}
onChange={handleInputChange(setUsername)}
/>
<br />
<TextField
type="password"
style={styles.textField}
error={!!errors.errorPassword}
helperText={errors.errorPassword}
label="Password"
name="password"
value={password}
onChange={handleInputChange(setPassword)}
/>
<FormGroup>
<FormControlLabel
control={
<Switch
color="primary"
checked={showAdvanced}
onChange={() => setShowAdvanced(!showAdvanced)}
/>
}
label="Advanced settings"
/>
</FormGroup>
{advancedSettings}
{errors.errorGeneral && (
<Alert severity="error" style={styles.textField}>{errors.errorGeneral}</Alert>
)}
{progress && (
<Alert severity="info" style={styles.textField}>{progress}</Alert>
)}
<div style={styles.submit}>
<Button
variant="contained"
color="secondary"
onClick={onSubmit}
disabled={loading}
>
{loading ? (
<CircularProgress />
) : "Migrate"
}
</Button>
</div>
</Container>
);
}

@ -29,6 +29,7 @@ import { addJournal, fetchAll, fetchEntries, fetchUserInfo, createUserInfo } fro
import { syncEntriesToItemMap } from "./journal-processors"; import { syncEntriesToItemMap } from "./journal-processors";
import { ContactType } from "./pim-types"; import { ContactType } from "./pim-types";
import { parseDate } from "./helpers"; import { parseDate } from "./helpers";
import MigrateV2 from "./MigrateV2";
export interface SyncInfoJournal { export interface SyncInfoJournal {
journal: EteSync.Journal; journal: EteSync.Journal;
@ -293,6 +294,16 @@ export default withRouter(function SyncGate(props: RouteComponentProps<{}> & Pro
/> />
)} )}
/> />
<Route
path={routeResolver.getRoute("migrate-v2")}
exact
render={() => (
<MigrateV2
etesync={etesync}
userInfo={userInfo}
/>
)}
/>
</Switch> </Switch>
); );
}); });

@ -147,3 +147,9 @@ export function parseDate(prop: ICAL.Property) {
return {}; return {};
} }
export function* arrayToChunkIterator<T>(arr: T[], size: number) {
for (let i = 0 ; i < arr.length ; i += size) {
yield arr.slice(i, i + size);
}
}

@ -52,6 +52,7 @@ export function colorIntToHtml(color?: number) {
} }
// tslint:disable:no-bitwise // tslint:disable:no-bitwise
color = color >>> 0;
const blue = color & 0xFF; const blue = color & 0xFF;
const green = (color >> 8) & 0xFF; const green = (color >> 8) & 0xFF;
const red = (color >> 16) & 0xFF; const red = (color >> 16) & 0xFF;

@ -12,6 +12,7 @@ export interface PimType {
uid: string; uid: string;
toIcal(): string; toIcal(): string;
clone(): PimType; clone(): PimType;
lastModified: ICAL.Time | undefined;
} }
export function timezoneLoadFromName(timezone: string | null) { export function timezoneLoadFromName(timezone: string | null) {
@ -372,6 +373,10 @@ export class ContactType implements PimType {
return this.comp.getFirstPropertyValue("bday"); return this.comp.getFirstPropertyValue("bday");
} }
get lastModified() {
return this.comp.getFirstPropertyValue("rev");
}
get group() { get group() {
const kind = this.comp.getFirstPropertyValue("kind"); const kind = this.comp.getFirstPropertyValue("kind");
return kind in ["group", "organization"]; return kind in ["group", "organization"];

@ -1477,6 +1477,11 @@
call-me-maybe "^1.0.1" call-me-maybe "^1.0.1"
glob-to-regexp "^0.3.0" glob-to-regexp "^0.3.0"
"@msgpack/msgpack@^1.12.2":
version "1.12.2"
resolved "https://registry.yarnpkg.com/@msgpack/msgpack/-/msgpack-1.12.2.tgz#6a22e99a49b131a8789053d0b0903834552da36f"
integrity sha512-Vwhc3ObxmDZmA5hY8mfsau2rJ4vGPvzbj20QSZ2/E1GDPF61QVyjLfNHak9xmel6pW4heRt3v1fHa6np9Ehfeg==
"@nodelib/fs.stat@^1.1.2": "@nodelib/fs.stat@^1.1.2":
version "1.1.3" version "1.1.3"
resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz#2b5a3ab3f918cca48a8c754c08168e3f03eba61b" resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz#2b5a3ab3f918cca48a8c754c08168e3f03eba61b"
@ -4636,6 +4641,16 @@ etag@~1.8.1:
resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=
etebase@^0.11.0:
version "0.11.0"
resolved "https://registry.yarnpkg.com/etebase/-/etebase-0.11.0.tgz#082e6785167957723419c8eac59372b70953a921"
integrity sha512-VZwYf/i9yy89TEm2vR2dREukVcfrLmnRPS6xo5WmOTZHqBXxxMfjXtIKbH9B4xpqT5khzHmKQW6y9qFCxmWzgg==
dependencies:
"@msgpack/msgpack" "^1.12.2"
libsodium-wrappers "0.7.6"
node-fetch "^2.6.0"
urijs "^1.19.1"
etesync@^0.3.1: etesync@^0.3.1:
version "0.3.1" version "0.3.1"
resolved "https://registry.yarnpkg.com/etesync/-/etesync-0.3.1.tgz#2af257d5617679177d93fb8f6bdaf64462aa15e0" resolved "https://registry.yarnpkg.com/etesync/-/etesync-0.3.1.tgz#2af257d5617679177d93fb8f6bdaf64462aa15e0"
@ -6965,6 +6980,18 @@ levn@^0.3.0, levn@~0.3.0:
prelude-ls "~1.1.2" prelude-ls "~1.1.2"
type-check "~0.3.2" type-check "~0.3.2"
libsodium-wrappers@0.7.6:
version "0.7.6"
resolved "https://registry.yarnpkg.com/libsodium-wrappers/-/libsodium-wrappers-0.7.6.tgz#baed4c16d4bf9610104875ad8a8e164d259d48fb"
integrity sha512-OUO2CWW5bHdLr6hkKLHIKI4raEkZrf3QHkhXsJ1yCh6MZ3JDA7jFD3kCATNquuGSG6MjjPHQIQms0y0gBDzjQg==
dependencies:
libsodium "0.7.6"
libsodium@0.7.6:
version "0.7.6"
resolved "https://registry.yarnpkg.com/libsodium/-/libsodium-0.7.6.tgz#018b80c5728054817845fbffa554274441bda277"
integrity sha512-hPb/04sEuLcTRdWDtd+xH3RXBihpmbPCsKW/Jtf4PsvdyKh+D6z2D2gvp/5BfoxseP+0FCOg66kE+0oGUE/loQ==
lie@3.1.1: lie@3.1.1:
version "3.1.1" version "3.1.1"
resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e" resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"
@ -7534,6 +7561,11 @@ no-case@^3.0.3:
lower-case "^2.0.1" lower-case "^2.0.1"
tslib "^1.10.0" tslib "^1.10.0"
node-fetch@^2.6.0:
version "2.6.0"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
node-forge@0.9.0: node-forge@0.9.0:
version "0.9.0" version "0.9.0"
resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.9.0.tgz#d624050edbb44874adca12bb9a52ec63cb782579" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.9.0.tgz#d624050edbb44874adca12bb9a52ec63cb782579"

Loading…
Cancel
Save