From f579f3fcb0068751f28e3ef6879a314c45812f53 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 31 Aug 2020 18:11:54 +0300 Subject: [PATCH] Migration: add a tool to migrate to etesync 2.0. --- package.json | 1 + src/App.tsx | 4 +- src/MigrateV2.tsx | 380 ++++++++++++++++++++++++++++++++++++++ src/SyncGate.tsx | 11 ++ src/helpers.tsx | 6 + src/journal-processors.ts | 1 + src/pim-types.ts | 5 + yarn.lock | 32 ++++ 8 files changed, 439 insertions(+), 1 deletion(-) create mode 100644 src/MigrateV2.tsx diff --git a/package.json b/package.json index 5439a05..0b6b565 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "@material-ui/lab": "^4.0.0-alpha.47", "@material-ui/pickers": "^3.2.10", "@material-ui/styles": "^4.6.0", + "etebase": "^0.11.0", "etesync": "^0.3.1", "fuse.js": "^5.0.9-beta", "ical.js": "^1.4.0", diff --git a/src/App.tsx b/src/App.tsx index ad3b508..36dd919 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -89,6 +89,8 @@ export const routeResolver = new RouteResolver({ }, debug: { }, + "migrate-v2": { + }, }); interface AppBarPropsType { @@ -143,7 +145,7 @@ function App(props: PropsType) { function autoRefresh() { if (navigator.onLine && props.credentials && - !(window.location.pathname.match(/.*\/(new|edit|duplicate)$/))) { + !(window.location.pathname.match(/.*\/(new|edit|duplicate)$/) || window.location.pathname.startsWith("/migrate-v2"))) { refresh(); } } diff --git a/src/MigrateV2.tsx b/src/MigrateV2.tsx new file mode 100644 index 0000000..8758a22 --- /dev/null +++ b/src/MigrateV2.tsx @@ -0,0 +1,380 @@ +// SPDX-FileCopyrightText: © 2017 EteSync Authors +// SPDX-License-Identifier: AGPL-3.0-only + +import * as React from "react"; +import * as EteSync from "etesync"; +import * as Etebase from "etebase"; +import Button from "@material-ui/core/Button"; +import TextField from "@material-ui/core/TextField"; +import Switch from "@material-ui/core/Switch"; +import { Map as ImmutableMap } from "immutable"; + +import ContactsIcon from "@material-ui/icons/Contacts"; +import CalendarTodayIcon from "@material-ui/icons/CalendarToday"; +import FormatListBulletedIcon from "@material-ui/icons/FormatListBulleted"; + +import Container from "./widgets/Container"; +import { useSelector } from "react-redux"; +import { StoreState, CredentialsData, UserInfoData } from "./store"; +import AppBarOverride from "./widgets/AppBarOverride"; +import ColorBox from "./widgets/ColorBox"; +import { List, ListItem } from "./widgets/List"; +import { colorIntToHtml } from "./journal-processors"; +import { Checkbox, FormGroup, FormControlLabel, CircularProgress } from "@material-ui/core"; +import Alert from "@material-ui/lab/Alert"; +import { arrayToChunkIterator } from "./helpers"; +import { ContactType, EventType, TaskType, PimType } from "./pim-types"; + +interface PropsType { + etesync: CredentialsData; + userInfo: UserInfoData; +} + +interface FormErrors { + errorEmail?: string; + errorPassword?: string; + errorServer?: string; + + errorGeneral?: string; +} + +export default function MigrateV2(props: PropsType) { + const [wantedJournals, setWantedJournals] = React.useState>(ImmutableMap({})); + const [username, setUsername] = React.useState(""); + const [password, setPassword] = React.useState(""); + const [server, setServer] = React.useState(""); + const [showAdvanced, setShowAdvanced] = React.useState(false); + const [loading, setLoading] = React.useState(false); + const [progress, setProgress] = React.useState(""); + const [errors, setErrors] = React.useState({}); + const journals = useSelector((state: StoreState) => state.cache.journals!); + const journalEntries = useSelector((state: StoreState) => state.cache.entries); + const derived = props.etesync.encryptionKey; + + const decryptedJournals = React.useMemo(() => { + return journals.map((journal) => { + const userInfo = props.userInfo; + const keyPair = userInfo.getKeyPair(userInfo.getCryptoManager(derived)); + const cryptoManager = journal.getCryptoManager(derived, keyPair); + const info = journal.getInfo(cryptoManager); + return { journal, info }; + }); + }, [props.userInfo, journals, derived]); + + const journalClicked = React.useCallback((journal: EteSync.Journal) => { + if (wantedJournals.has(journal.uid)) { + setWantedJournals(wantedJournals.remove(journal.uid)); + } else { + setWantedJournals(wantedJournals.set(journal.uid, journal)); + } + }, [wantedJournals]); + + const journalMap = React.useMemo(() => { + return decryptedJournals.reduce( + (ret, { journal, info }) => { + let colorBox: React.ReactElement | undefined; + switch (info.type) { + case "CALENDAR": + case "TASKS": + colorBox = ( + + ); + break; + } + ret[info.type] = ret[info.type] || []; + ret[info.type].push( + journalClicked(journal)}> + + {info.displayName} + + ); + + return ret; + }, + { + CALENDAR: [], + ADDRESS_BOOK: [], + TASKS: [], + }); + }, [decryptedJournals, journalClicked]); + + const styles = { + form: { + }, + forgotPassword: { + paddingTop: 20, + }, + textField: { + marginTop: 20, + }, + submit: { + marginTop: 40, + }, + }; + + function handleInputChange(func: (value: string) => void) { + return (event: React.ChangeEvent) => { + func(event.target.value); + }; + } + + async function onSubmit() { + setLoading(true); + setProgress(""); + try { + const errors: FormErrors = {}; + const fieldRequired = "This field is required!"; + if (!username) { + errors.errorEmail = fieldRequired; + } + if (!password) { + errors.errorPassword = fieldRequired; + } + if (showAdvanced && !server.startsWith("http")) { + errors.errorServer = "Server URI must start with http/https://"; + } + + if (Object.keys(errors).length) { + setErrors(errors); + return; + } else { + setErrors({}); + } + + const serverUrl = (showAdvanced) ? server : undefined; + let malformed = 0; + + setProgress("Logging into EteSync 2.0 account"); + const etebase = await Etebase.Account.login(username, password, serverUrl); + const colMgr = etebase.getCollectionManager(); + + const { etesync, userInfo } = props; + const derived = etesync.encryptionKey; + const userInfoCryptoManager = userInfo.getCryptoManager(etesync.encryptionKey); + const keyPair = userInfo.getKeyPair(userInfoCryptoManager); + + const now = (new Date()).getTime(); + + let i = 1; + for (const journal of wantedJournals.values()) { + setProgress(`Migrating collection ${i}/${wantedJournals.size}`); + console.log("Migrating ", journal.uid); + + const cryptoManager = journal.getCryptoManager(derived, keyPair); + const info = journal.getInfo(cryptoManager); + const entries = journalEntries.get(journal.uid)!; + + let colType; + let parseFunc: (content: string) => PimType; + switch (info.type) { + case "ADDRESS_BOOK": { + colType = "etebase.vcard"; + parseFunc = ContactType.parse; + break; + } + case "CALENDAR": { + colType = "etebase.vevent"; + parseFunc = EventType.parse; + break; + } + case "TASKS": { + colType = "etebase.vtodo"; + parseFunc = TaskType.parse; + break; + } + default: { + continue; + } + } + const meta: Etebase.CollectionMetadata = { + type: colType, + name: info.displayName, + description: info.description, + color: (info.color !== undefined) ? colorIntToHtml(info.color) : undefined, + }; + const collection = await colMgr.create(meta, ""); + await colMgr.upload(collection); + + const itemMgr = colMgr.getItemManager(collection); + + const CHUNK_SIZE = 20; + const items = new Map(); + let done = 0; + let prevUid: string | null = null; + for (const chunk of arrayToChunkIterator(entries.toArray(), CHUNK_SIZE)) { + setProgress(`Migrating collection ${i}/${wantedJournals.size}\nMigrated entries: ${done}/${entries.size}`); + const chunkItems = []; + for (const entry of chunk) { + console.log("Migrating entry ", entry.uid); + done++; + const syncEntry = entry.getSyncEntry(cryptoManager, prevUid); + prevUid = entry.uid; + const pimItem = parseFunc(syncEntry.content); + const uid = pimItem.uid; + // When we can't set mtime, set to the item's position in the change log so we at least maintain EteSync 1.0 ordering. + const mtime = (pimItem.lastModified?.toJSDate())?.getTime() ?? now + done; + + if (!uid) { + malformed++; + continue; + } + + let item = items.get(uid); + if (item) { + // Existing item + item = item._clone(); // We are cloning so we can push multiple revisions at once + await item.setContent(syncEntry.content); + const meta = await item.getMeta(); + meta.mtime = mtime; + await item.setMeta(meta); + } else { + // New + const meta: Etebase.ItemMetadata = { + mtime, + name: uid, + }; + item = await itemMgr.create(meta, syncEntry.content); + items.set(uid, item); + } + if (syncEntry.action === EteSync.SyncEntryAction.Delete) { + item.delete(true); + } + + chunkItems.push(item); + } + await itemMgr.batch(chunkItems); + } + + i++; + } + + await etebase.logout(); + if (malformed > 0) { + setProgress(`Done\nIgnored ${malformed} entries (probably safe to ignore)`); + } else { + setProgress("Done"); + } + + } catch (e) { + if (e instanceof Etebase.UnauthorizedError) { + errors.errorGeneral = "Wrong username or password"; + } else { + errors.errorGeneral = e.toString(); + } + setErrors(errors); + } finally { + setLoading(false); + } + } + + let advancedSettings = null; + if (showAdvanced) { + advancedSettings = ( + + +
+ + ); + } + + return ( + + +

+ This tool will help you migrate your data to EteSync 2.0. +

+

+ The migration doesn't delete any data. It only copies your data over to the new EteSync 2.0 server. This means that there is no risk of data-loss in the migration. +

+

+ Please select the collections you would like to migrate, and then enter your EteSync 2.0 credentials and click migrate. +

+ + } + nestedItems={journalMap.ADDRESS_BOOK} + /> + + } + nestedItems={journalMap.CALENDAR} + /> + + } + nestedItems={journalMap.TASKS} + /> + + +

EteSync 2.0 credentials

+ +
+ + + setShowAdvanced(!showAdvanced)} + /> + } + label="Advanced settings" + /> + + {advancedSettings} + {errors.errorGeneral && ( + {errors.errorGeneral} + )} + {progress && ( + {progress} + )} +
+ +
+ + ); +} diff --git a/src/SyncGate.tsx b/src/SyncGate.tsx index f4de697..16d9987 100644 --- a/src/SyncGate.tsx +++ b/src/SyncGate.tsx @@ -29,6 +29,7 @@ import { addJournal, fetchAll, fetchEntries, fetchUserInfo, createUserInfo } fro import { syncEntriesToItemMap } from "./journal-processors"; import { ContactType } from "./pim-types"; import { parseDate } from "./helpers"; +import MigrateV2 from "./MigrateV2"; export interface SyncInfoJournal { journal: EteSync.Journal; @@ -293,6 +294,16 @@ export default withRouter(function SyncGate(props: RouteComponentProps<{}> & Pro /> )} /> + ( + + )} + /> ); }); diff --git a/src/helpers.tsx b/src/helpers.tsx index 91da9ea..993b84e 100644 --- a/src/helpers.tsx +++ b/src/helpers.tsx @@ -147,3 +147,9 @@ export function parseDate(prop: ICAL.Property) { return {}; } + +export function* arrayToChunkIterator(arr: T[], size: number) { + for (let i = 0 ; i < arr.length ; i += size) { + yield arr.slice(i, i + size); + } +} diff --git a/src/journal-processors.ts b/src/journal-processors.ts index 9e7f314..cfeaef6 100644 --- a/src/journal-processors.ts +++ b/src/journal-processors.ts @@ -52,6 +52,7 @@ export function colorIntToHtml(color?: number) { } // tslint:disable:no-bitwise + color = color >>> 0; const blue = color & 0xFF; const green = (color >> 8) & 0xFF; const red = (color >> 16) & 0xFF; diff --git a/src/pim-types.ts b/src/pim-types.ts index fd40b33..b196bcd 100644 --- a/src/pim-types.ts +++ b/src/pim-types.ts @@ -12,6 +12,7 @@ export interface PimType { uid: string; toIcal(): string; clone(): PimType; + lastModified: ICAL.Time | undefined; } export function timezoneLoadFromName(timezone: string | null) { @@ -372,6 +373,10 @@ export class ContactType implements PimType { return this.comp.getFirstPropertyValue("bday"); } + get lastModified() { + return this.comp.getFirstPropertyValue("rev"); + } + get group() { const kind = this.comp.getFirstPropertyValue("kind"); return kind in ["group", "organization"]; diff --git a/yarn.lock b/yarn.lock index abb28df..99e7854 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1477,6 +1477,11 @@ call-me-maybe "^1.0.1" glob-to-regexp "^0.3.0" +"@msgpack/msgpack@^1.12.2": + version "1.12.2" + resolved "https://registry.yarnpkg.com/@msgpack/msgpack/-/msgpack-1.12.2.tgz#6a22e99a49b131a8789053d0b0903834552da36f" + integrity sha512-Vwhc3ObxmDZmA5hY8mfsau2rJ4vGPvzbj20QSZ2/E1GDPF61QVyjLfNHak9xmel6pW4heRt3v1fHa6np9Ehfeg== + "@nodelib/fs.stat@^1.1.2": version "1.1.3" resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz#2b5a3ab3f918cca48a8c754c08168e3f03eba61b" @@ -4636,6 +4641,16 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= +etebase@^0.11.0: + version "0.11.0" + resolved "https://registry.yarnpkg.com/etebase/-/etebase-0.11.0.tgz#082e6785167957723419c8eac59372b70953a921" + integrity sha512-VZwYf/i9yy89TEm2vR2dREukVcfrLmnRPS6xo5WmOTZHqBXxxMfjXtIKbH9B4xpqT5khzHmKQW6y9qFCxmWzgg== + dependencies: + "@msgpack/msgpack" "^1.12.2" + libsodium-wrappers "0.7.6" + node-fetch "^2.6.0" + urijs "^1.19.1" + etesync@^0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/etesync/-/etesync-0.3.1.tgz#2af257d5617679177d93fb8f6bdaf64462aa15e0" @@ -6965,6 +6980,18 @@ levn@^0.3.0, levn@~0.3.0: prelude-ls "~1.1.2" type-check "~0.3.2" +libsodium-wrappers@0.7.6: + version "0.7.6" + resolved "https://registry.yarnpkg.com/libsodium-wrappers/-/libsodium-wrappers-0.7.6.tgz#baed4c16d4bf9610104875ad8a8e164d259d48fb" + integrity sha512-OUO2CWW5bHdLr6hkKLHIKI4raEkZrf3QHkhXsJ1yCh6MZ3JDA7jFD3kCATNquuGSG6MjjPHQIQms0y0gBDzjQg== + dependencies: + libsodium "0.7.6" + +libsodium@0.7.6: + version "0.7.6" + resolved "https://registry.yarnpkg.com/libsodium/-/libsodium-0.7.6.tgz#018b80c5728054817845fbffa554274441bda277" + integrity sha512-hPb/04sEuLcTRdWDtd+xH3RXBihpmbPCsKW/Jtf4PsvdyKh+D6z2D2gvp/5BfoxseP+0FCOg66kE+0oGUE/loQ== + lie@3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e" @@ -7534,6 +7561,11 @@ no-case@^3.0.3: lower-case "^2.0.1" tslib "^1.10.0" +node-fetch@^2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" + integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== + node-forge@0.9.0: version "0.9.0" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.9.0.tgz#d624050edbb44874adca12bb9a52ec63cb782579"