Migration: add a tool to migrate to etesync 2.0.
Tom Hacohen
parent
5a5c777619
commit
f579f3fcb0
|
|
@ -9,6 +9,7 @@
|
|||
"@material-ui/lab": "^4.0.0-alpha.47",
|
||||
"@material-ui/pickers": "^3.2.10",
|
||||
"@material-ui/styles": "^4.6.0",
|
||||
"etebase": "^0.11.0",
|
||||
"etesync": "^0.3.1",
|
||||
"fuse.js": "^5.0.9-beta",
|
||||
"ical.js": "^1.4.0",
|
||||
|
|
|
|||
|
|
@ -89,6 +89,8 @@ export const routeResolver = new RouteResolver({
|
|||
},
|
||||
debug: {
|
||||
},
|
||||
"migrate-v2": {
|
||||
},
|
||||
});
|
||||
|
||||
interface AppBarPropsType {
|
||||
|
|
@ -143,7 +145,7 @@ function App(props: PropsType) {
|
|||
|
||||
function autoRefresh() {
|
||||
if (navigator.onLine && props.credentials &&
|
||||
!(window.location.pathname.match(/.*\/(new|edit|duplicate)$/))) {
|
||||
!(window.location.pathname.match(/.*\/(new|edit|duplicate)$/) || window.location.pathname.startsWith("/migrate-v2"))) {
|
||||
refresh();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,380 @@
|
|||
// SPDX-FileCopyrightText: © 2017 EteSync Authors
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import * as React from "react";
|
||||
import * as EteSync from "etesync";
|
||||
import * as Etebase from "etebase";
|
||||
import Button from "@material-ui/core/Button";
|
||||
import TextField from "@material-ui/core/TextField";
|
||||
import Switch from "@material-ui/core/Switch";
|
||||
import { Map as ImmutableMap } from "immutable";
|
||||
|
||||
import ContactsIcon from "@material-ui/icons/Contacts";
|
||||
import CalendarTodayIcon from "@material-ui/icons/CalendarToday";
|
||||
import FormatListBulletedIcon from "@material-ui/icons/FormatListBulleted";
|
||||
|
||||
import Container from "./widgets/Container";
|
||||
import { useSelector } from "react-redux";
|
||||
import { StoreState, CredentialsData, UserInfoData } from "./store";
|
||||
import AppBarOverride from "./widgets/AppBarOverride";
|
||||
import ColorBox from "./widgets/ColorBox";
|
||||
import { List, ListItem } from "./widgets/List";
|
||||
import { colorIntToHtml } from "./journal-processors";
|
||||
import { Checkbox, FormGroup, FormControlLabel, CircularProgress } from "@material-ui/core";
|
||||
import Alert from "@material-ui/lab/Alert";
|
||||
import { arrayToChunkIterator } from "./helpers";
|
||||
import { ContactType, EventType, TaskType, PimType } from "./pim-types";
|
||||
|
||||
interface PropsType {
|
||||
etesync: CredentialsData;
|
||||
userInfo: UserInfoData;
|
||||
}
|
||||
|
||||
interface FormErrors {
|
||||
errorEmail?: string;
|
||||
errorPassword?: string;
|
||||
errorServer?: string;
|
||||
|
||||
errorGeneral?: string;
|
||||
}
|
||||
|
||||
export default function MigrateV2(props: PropsType) {
|
||||
const [wantedJournals, setWantedJournals] = React.useState<ImmutableMap<string, EteSync.Journal>>(ImmutableMap({}));
|
||||
const [username, setUsername] = React.useState("");
|
||||
const [password, setPassword] = React.useState("");
|
||||
const [server, setServer] = React.useState("");
|
||||
const [showAdvanced, setShowAdvanced] = React.useState(false);
|
||||
const [loading, setLoading] = React.useState(false);
|
||||
const [progress, setProgress] = React.useState("");
|
||||
const [errors, setErrors] = React.useState<FormErrors>({});
|
||||
const journals = useSelector((state: StoreState) => state.cache.journals!);
|
||||
const journalEntries = useSelector((state: StoreState) => state.cache.entries);
|
||||
const derived = props.etesync.encryptionKey;
|
||||
|
||||
const decryptedJournals = React.useMemo(() => {
|
||||
return journals.map((journal) => {
|
||||
const userInfo = props.userInfo;
|
||||
const keyPair = userInfo.getKeyPair(userInfo.getCryptoManager(derived));
|
||||
const cryptoManager = journal.getCryptoManager(derived, keyPair);
|
||||
const info = journal.getInfo(cryptoManager);
|
||||
return { journal, info };
|
||||
});
|
||||
}, [props.userInfo, journals, derived]);
|
||||
|
||||
const journalClicked = React.useCallback((journal: EteSync.Journal) => {
|
||||
if (wantedJournals.has(journal.uid)) {
|
||||
setWantedJournals(wantedJournals.remove(journal.uid));
|
||||
} else {
|
||||
setWantedJournals(wantedJournals.set(journal.uid, journal));
|
||||
}
|
||||
}, [wantedJournals]);
|
||||
|
||||
const journalMap = React.useMemo(() => {
|
||||
return decryptedJournals.reduce(
|
||||
(ret, { journal, info }) => {
|
||||
let colorBox: React.ReactElement | undefined;
|
||||
switch (info.type) {
|
||||
case "CALENDAR":
|
||||
case "TASKS":
|
||||
colorBox = (
|
||||
<ColorBox size={24} color={colorIntToHtml(info.color)} />
|
||||
);
|
||||
break;
|
||||
}
|
||||
ret[info.type] = ret[info.type] || [];
|
||||
ret[info.type].push(
|
||||
<ListItem key={journal.uid} rightIcon={colorBox} insetChildren
|
||||
onClick={() => journalClicked(journal)}>
|
||||
<Checkbox
|
||||
checked={wantedJournals.has(journal.uid)}
|
||||
/>
|
||||
{info.displayName}
|
||||
</ListItem>
|
||||
);
|
||||
|
||||
return ret;
|
||||
},
|
||||
{
|
||||
CALENDAR: [],
|
||||
ADDRESS_BOOK: [],
|
||||
TASKS: [],
|
||||
});
|
||||
}, [decryptedJournals, journalClicked]);
|
||||
|
||||
const styles = {
|
||||
form: {
|
||||
},
|
||||
forgotPassword: {
|
||||
paddingTop: 20,
|
||||
},
|
||||
textField: {
|
||||
marginTop: 20,
|
||||
},
|
||||
submit: {
|
||||
marginTop: 40,
|
||||
},
|
||||
};
|
||||
|
||||
function handleInputChange(func: (value: string) => void) {
|
||||
return (event: React.ChangeEvent<any>) => {
|
||||
func(event.target.value);
|
||||
};
|
||||
}
|
||||
|
||||
async function onSubmit() {
|
||||
setLoading(true);
|
||||
setProgress("");
|
||||
try {
|
||||
const errors: FormErrors = {};
|
||||
const fieldRequired = "This field is required!";
|
||||
if (!username) {
|
||||
errors.errorEmail = fieldRequired;
|
||||
}
|
||||
if (!password) {
|
||||
errors.errorPassword = fieldRequired;
|
||||
}
|
||||
if (showAdvanced && !server.startsWith("http")) {
|
||||
errors.errorServer = "Server URI must start with http/https://";
|
||||
}
|
||||
|
||||
if (Object.keys(errors).length) {
|
||||
setErrors(errors);
|
||||
return;
|
||||
} else {
|
||||
setErrors({});
|
||||
}
|
||||
|
||||
const serverUrl = (showAdvanced) ? server : undefined;
|
||||
let malformed = 0;
|
||||
|
||||
setProgress("Logging into EteSync 2.0 account");
|
||||
const etebase = await Etebase.Account.login(username, password, serverUrl);
|
||||
const colMgr = etebase.getCollectionManager();
|
||||
|
||||
const { etesync, userInfo } = props;
|
||||
const derived = etesync.encryptionKey;
|
||||
const userInfoCryptoManager = userInfo.getCryptoManager(etesync.encryptionKey);
|
||||
const keyPair = userInfo.getKeyPair(userInfoCryptoManager);
|
||||
|
||||
const now = (new Date()).getTime();
|
||||
|
||||
let i = 1;
|
||||
for (const journal of wantedJournals.values()) {
|
||||
setProgress(`Migrating collection ${i}/${wantedJournals.size}`);
|
||||
console.log("Migrating ", journal.uid);
|
||||
|
||||
const cryptoManager = journal.getCryptoManager(derived, keyPair);
|
||||
const info = journal.getInfo(cryptoManager);
|
||||
const entries = journalEntries.get(journal.uid)!;
|
||||
|
||||
let colType;
|
||||
let parseFunc: (content: string) => PimType;
|
||||
switch (info.type) {
|
||||
case "ADDRESS_BOOK": {
|
||||
colType = "etebase.vcard";
|
||||
parseFunc = ContactType.parse;
|
||||
break;
|
||||
}
|
||||
case "CALENDAR": {
|
||||
colType = "etebase.vevent";
|
||||
parseFunc = EventType.parse;
|
||||
break;
|
||||
}
|
||||
case "TASKS": {
|
||||
colType = "etebase.vtodo";
|
||||
parseFunc = TaskType.parse;
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const meta: Etebase.CollectionMetadata = {
|
||||
type: colType,
|
||||
name: info.displayName,
|
||||
description: info.description,
|
||||
color: (info.color !== undefined) ? colorIntToHtml(info.color) : undefined,
|
||||
};
|
||||
const collection = await colMgr.create(meta, "");
|
||||
await colMgr.upload(collection);
|
||||
|
||||
const itemMgr = colMgr.getItemManager(collection);
|
||||
|
||||
const CHUNK_SIZE = 20;
|
||||
const items = new Map<string, Etebase.Item>();
|
||||
let done = 0;
|
||||
let prevUid: string | null = null;
|
||||
for (const chunk of arrayToChunkIterator(entries.toArray(), CHUNK_SIZE)) {
|
||||
setProgress(`Migrating collection ${i}/${wantedJournals.size}\nMigrated entries: ${done}/${entries.size}`);
|
||||
const chunkItems = [];
|
||||
for (const entry of chunk) {
|
||||
console.log("Migrating entry ", entry.uid);
|
||||
done++;
|
||||
const syncEntry = entry.getSyncEntry(cryptoManager, prevUid);
|
||||
prevUid = entry.uid;
|
||||
const pimItem = parseFunc(syncEntry.content);
|
||||
const uid = pimItem.uid;
|
||||
// When we can't set mtime, set to the item's position in the change log so we at least maintain EteSync 1.0 ordering.
|
||||
const mtime = (pimItem.lastModified?.toJSDate())?.getTime() ?? now + done;
|
||||
|
||||
if (!uid) {
|
||||
malformed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
let item = items.get(uid);
|
||||
if (item) {
|
||||
// Existing item
|
||||
item = item._clone(); // We are cloning so we can push multiple revisions at once
|
||||
await item.setContent(syncEntry.content);
|
||||
const meta = await item.getMeta();
|
||||
meta.mtime = mtime;
|
||||
await item.setMeta(meta);
|
||||
} else {
|
||||
// New
|
||||
const meta: Etebase.ItemMetadata = {
|
||||
mtime,
|
||||
name: uid,
|
||||
};
|
||||
item = await itemMgr.create(meta, syncEntry.content);
|
||||
items.set(uid, item);
|
||||
}
|
||||
if (syncEntry.action === EteSync.SyncEntryAction.Delete) {
|
||||
item.delete(true);
|
||||
}
|
||||
|
||||
chunkItems.push(item);
|
||||
}
|
||||
await itemMgr.batch(chunkItems);
|
||||
}
|
||||
|
||||
i++;
|
||||
}
|
||||
|
||||
await etebase.logout();
|
||||
if (malformed > 0) {
|
||||
setProgress(`Done\nIgnored ${malformed} entries (probably safe to ignore)`);
|
||||
} else {
|
||||
setProgress("Done");
|
||||
}
|
||||
|
||||
} catch (e) {
|
||||
if (e instanceof Etebase.UnauthorizedError) {
|
||||
errors.errorGeneral = "Wrong username or password";
|
||||
} else {
|
||||
errors.errorGeneral = e.toString();
|
||||
}
|
||||
setErrors(errors);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
let advancedSettings = null;
|
||||
if (showAdvanced) {
|
||||
advancedSettings = (
|
||||
<React.Fragment>
|
||||
<TextField
|
||||
type="url"
|
||||
style={styles.textField}
|
||||
error={!!errors.errorServer}
|
||||
helperText={errors.errorServer}
|
||||
label="Server"
|
||||
name="server"
|
||||
value={server}
|
||||
onChange={handleInputChange(setServer)}
|
||||
/>
|
||||
<br />
|
||||
</React.Fragment>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Container>
|
||||
<AppBarOverride title="Migrate to EteSync 2.0" />
|
||||
<p>
|
||||
This tool will help you migrate your data to EteSync 2.0.
|
||||
</p>
|
||||
<p>
|
||||
The migration doesn't delete any data. It only copies your data over to the new EteSync 2.0 server. This means that there is no risk of data-loss in the migration.
|
||||
</p>
|
||||
<p>
|
||||
Please select the collections you would like to migrate, and then enter your EteSync 2.0 credentials and click migrate.
|
||||
</p>
|
||||
<List>
|
||||
<ListItem
|
||||
primaryText="Address Books"
|
||||
leftIcon={<ContactsIcon />}
|
||||
nestedItems={journalMap.ADDRESS_BOOK}
|
||||
/>
|
||||
|
||||
<ListItem
|
||||
primaryText="Calendars"
|
||||
leftIcon={<CalendarTodayIcon />}
|
||||
nestedItems={journalMap.CALENDAR}
|
||||
/>
|
||||
|
||||
<ListItem
|
||||
primaryText="Tasks"
|
||||
leftIcon={<FormatListBulletedIcon />}
|
||||
nestedItems={journalMap.TASKS}
|
||||
/>
|
||||
</List>
|
||||
|
||||
<h3>EteSync 2.0 credentials</h3>
|
||||
<TextField
|
||||
type="text"
|
||||
style={styles.textField}
|
||||
error={!!errors.errorEmail}
|
||||
helperText={errors.errorEmail}
|
||||
label="Username"
|
||||
name="username"
|
||||
value={username}
|
||||
onChange={handleInputChange(setUsername)}
|
||||
/>
|
||||
<br />
|
||||
<TextField
|
||||
type="password"
|
||||
style={styles.textField}
|
||||
error={!!errors.errorPassword}
|
||||
helperText={errors.errorPassword}
|
||||
label="Password"
|
||||
name="password"
|
||||
value={password}
|
||||
onChange={handleInputChange(setPassword)}
|
||||
/>
|
||||
<FormGroup>
|
||||
<FormControlLabel
|
||||
control={
|
||||
<Switch
|
||||
color="primary"
|
||||
checked={showAdvanced}
|
||||
onChange={() => setShowAdvanced(!showAdvanced)}
|
||||
/>
|
||||
}
|
||||
label="Advanced settings"
|
||||
/>
|
||||
</FormGroup>
|
||||
{advancedSettings}
|
||||
{errors.errorGeneral && (
|
||||
<Alert severity="error" style={styles.textField}>{errors.errorGeneral}</Alert>
|
||||
)}
|
||||
{progress && (
|
||||
<Alert severity="info" style={styles.textField}>{progress}</Alert>
|
||||
)}
|
||||
<div style={styles.submit}>
|
||||
<Button
|
||||
variant="contained"
|
||||
color="secondary"
|
||||
onClick={onSubmit}
|
||||
disabled={loading}
|
||||
>
|
||||
{loading ? (
|
||||
<CircularProgress />
|
||||
) : "Migrate"
|
||||
}
|
||||
</Button>
|
||||
</div>
|
||||
</Container>
|
||||
);
|
||||
}
|
||||
|
|
@ -29,6 +29,7 @@ import { addJournal, fetchAll, fetchEntries, fetchUserInfo, createUserInfo } fro
|
|||
import { syncEntriesToItemMap } from "./journal-processors";
|
||||
import { ContactType } from "./pim-types";
|
||||
import { parseDate } from "./helpers";
|
||||
import MigrateV2 from "./MigrateV2";
|
||||
|
||||
export interface SyncInfoJournal {
|
||||
journal: EteSync.Journal;
|
||||
|
|
@ -293,6 +294,16 @@ export default withRouter(function SyncGate(props: RouteComponentProps<{}> & Pro
|
|||
/>
|
||||
)}
|
||||
/>
|
||||
<Route
|
||||
path={routeResolver.getRoute("migrate-v2")}
|
||||
exact
|
||||
render={() => (
|
||||
<MigrateV2
|
||||
etesync={etesync}
|
||||
userInfo={userInfo}
|
||||
/>
|
||||
)}
|
||||
/>
|
||||
</Switch>
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -147,3 +147,9 @@ export function parseDate(prop: ICAL.Property) {
|
|||
|
||||
return {};
|
||||
}
|
||||
|
||||
export function* arrayToChunkIterator<T>(arr: T[], size: number) {
|
||||
for (let i = 0 ; i < arr.length ; i += size) {
|
||||
yield arr.slice(i, i + size);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ export function colorIntToHtml(color?: number) {
|
|||
}
|
||||
|
||||
// tslint:disable:no-bitwise
|
||||
color = color >>> 0;
|
||||
const blue = color & 0xFF;
|
||||
const green = (color >> 8) & 0xFF;
|
||||
const red = (color >> 16) & 0xFF;
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ export interface PimType {
|
|||
uid: string;
|
||||
toIcal(): string;
|
||||
clone(): PimType;
|
||||
lastModified: ICAL.Time | undefined;
|
||||
}
|
||||
|
||||
export function timezoneLoadFromName(timezone: string | null) {
|
||||
|
|
@ -372,6 +373,10 @@ export class ContactType implements PimType {
|
|||
return this.comp.getFirstPropertyValue("bday");
|
||||
}
|
||||
|
||||
get lastModified() {
|
||||
return this.comp.getFirstPropertyValue("rev");
|
||||
}
|
||||
|
||||
get group() {
|
||||
const kind = this.comp.getFirstPropertyValue("kind");
|
||||
return kind in ["group", "organization"];
|
||||
|
|
|
|||
32
yarn.lock
32
yarn.lock
|
|
@ -1477,6 +1477,11 @@
|
|||
call-me-maybe "^1.0.1"
|
||||
glob-to-regexp "^0.3.0"
|
||||
|
||||
"@msgpack/msgpack@^1.12.2":
|
||||
version "1.12.2"
|
||||
resolved "https://registry.yarnpkg.com/@msgpack/msgpack/-/msgpack-1.12.2.tgz#6a22e99a49b131a8789053d0b0903834552da36f"
|
||||
integrity sha512-Vwhc3ObxmDZmA5hY8mfsau2rJ4vGPvzbj20QSZ2/E1GDPF61QVyjLfNHak9xmel6pW4heRt3v1fHa6np9Ehfeg==
|
||||
|
||||
"@nodelib/fs.stat@^1.1.2":
|
||||
version "1.1.3"
|
||||
resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz#2b5a3ab3f918cca48a8c754c08168e3f03eba61b"
|
||||
|
|
@ -4636,6 +4641,16 @@ etag@~1.8.1:
|
|||
resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
|
||||
integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=
|
||||
|
||||
etebase@^0.11.0:
|
||||
version "0.11.0"
|
||||
resolved "https://registry.yarnpkg.com/etebase/-/etebase-0.11.0.tgz#082e6785167957723419c8eac59372b70953a921"
|
||||
integrity sha512-VZwYf/i9yy89TEm2vR2dREukVcfrLmnRPS6xo5WmOTZHqBXxxMfjXtIKbH9B4xpqT5khzHmKQW6y9qFCxmWzgg==
|
||||
dependencies:
|
||||
"@msgpack/msgpack" "^1.12.2"
|
||||
libsodium-wrappers "0.7.6"
|
||||
node-fetch "^2.6.0"
|
||||
urijs "^1.19.1"
|
||||
|
||||
etesync@^0.3.1:
|
||||
version "0.3.1"
|
||||
resolved "https://registry.yarnpkg.com/etesync/-/etesync-0.3.1.tgz#2af257d5617679177d93fb8f6bdaf64462aa15e0"
|
||||
|
|
@ -6965,6 +6980,18 @@ levn@^0.3.0, levn@~0.3.0:
|
|||
prelude-ls "~1.1.2"
|
||||
type-check "~0.3.2"
|
||||
|
||||
libsodium-wrappers@0.7.6:
|
||||
version "0.7.6"
|
||||
resolved "https://registry.yarnpkg.com/libsodium-wrappers/-/libsodium-wrappers-0.7.6.tgz#baed4c16d4bf9610104875ad8a8e164d259d48fb"
|
||||
integrity sha512-OUO2CWW5bHdLr6hkKLHIKI4raEkZrf3QHkhXsJ1yCh6MZ3JDA7jFD3kCATNquuGSG6MjjPHQIQms0y0gBDzjQg==
|
||||
dependencies:
|
||||
libsodium "0.7.6"
|
||||
|
||||
libsodium@0.7.6:
|
||||
version "0.7.6"
|
||||
resolved "https://registry.yarnpkg.com/libsodium/-/libsodium-0.7.6.tgz#018b80c5728054817845fbffa554274441bda277"
|
||||
integrity sha512-hPb/04sEuLcTRdWDtd+xH3RXBihpmbPCsKW/Jtf4PsvdyKh+D6z2D2gvp/5BfoxseP+0FCOg66kE+0oGUE/loQ==
|
||||
|
||||
lie@3.1.1:
|
||||
version "3.1.1"
|
||||
resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"
|
||||
|
|
@ -7534,6 +7561,11 @@ no-case@^3.0.3:
|
|||
lower-case "^2.0.1"
|
||||
tslib "^1.10.0"
|
||||
|
||||
node-fetch@^2.6.0:
|
||||
version "2.6.0"
|
||||
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
|
||||
integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
|
||||
|
||||
node-forge@0.9.0:
|
||||
version "0.9.0"
|
||||
resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.9.0.tgz#d624050edbb44874adca12bb9a52ec63cb782579"
|
||||
|
|
|
|||
Loading…
Reference in New Issue