From 1f1d3b6a8953639f86c4c26eef2ecadef20ef13c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 5 Aug 2020 12:05:23 +0300 Subject: [PATCH] basic sync --- src/LoginGate.tsx | 12 ++- src/SyncGate.tsx | 223 ++++++++++------------------------------ src/sync/SyncManager.ts | 131 +++++++++++++++++++++++ 3 files changed, 192 insertions(+), 174 deletions(-) create mode 100644 src/sync/SyncManager.ts diff --git a/src/LoginGate.tsx b/src/LoginGate.tsx index f98e6a0..02e5e58 100644 --- a/src/LoginGate.tsx +++ b/src/LoginGate.tsx @@ -16,6 +16,7 @@ import * as C from "./constants"; import SignedPagesBadge from "./images/signed-pages-badge.svg"; import { useCredentials } from "./credentials"; +import LoadingIndicator from "./widgets/LoadingIndicator"; export default function LoginGate() { @@ -35,9 +36,11 @@ export default function LoginGate() { } } - const loading = credentials === undefined; - - if (!credentials) { + if (credentials === undefined) { + return ( + + ); + } else if (credentials === null) { const style = { isSafe: { textDecoration: "none", @@ -55,7 +58,6 @@ export default function LoginGate() {
@@ -75,6 +77,6 @@ export default function LoginGate() { } return ( - + ); } diff --git a/src/SyncGate.tsx b/src/SyncGate.tsx index b6a62c8..17f0b3e 100644 --- a/src/SyncGate.tsx +++ b/src/SyncGate.tsx @@ -2,14 +2,13 @@ // SPDX-License-Identifier: AGPL-3.0-only import * as React from "react"; -import { useSelector } from "react-redux"; -import { Route, Switch, Redirect, RouteComponentProps, withRouter } from "react-router"; +import { useSelector, useDispatch } from "react-redux"; +import { Route, Switch, Redirect, withRouter, useHistory } from "react-router"; import moment from "moment"; import "moment/locale/en-gb"; import { List, Map } from "immutable"; -import { createSelector } from "reselect"; import { routeResolver } from "./App"; @@ -22,10 +21,12 @@ import Debug from "./Debug"; import Pim from "./Pim"; import * as EteSync from "etesync"; -import { CURRENT_VERSION } from "etesync"; -import { store, JournalsData, EntriesData, StoreState, CredentialsData, UserInfoData } from "./store"; -import { addJournal, fetchAll, fetchEntries, fetchUserInfo, createUserInfo } from "./store/actions"; +import { SyncManager } from "./sync/SyncManager"; + +import { StoreState } from "./store"; +import { performSync } from "./store/actions"; +import { useCredentials } from "./credentials"; export interface SyncInfoJournal { journal: EteSync.Journal; @@ -36,157 +37,37 @@ export interface SyncInfoJournal { export type SyncInfo = Map; -interface PropsType { - etesync: CredentialsData; -} - -interface SelectorProps { - etesync: CredentialsData; - journals: JournalsData; - entries: EntriesData; - userInfo: UserInfoData; -} - -const syncInfoSelector = createSelector( - (props: SelectorProps) => props.etesync, - (props: SelectorProps) => props.journals!, - (props: SelectorProps) => props.entries, - (props: SelectorProps) => props.userInfo, - (etesync, journals, entries, userInfo) => { - const derived = etesync.encryptionKey; - const userInfoCryptoManager = userInfo.getCryptoManager(etesync.encryptionKey); - try { - userInfo.verify(userInfoCryptoManager); - } catch (error) { - if (error instanceof EteSync.IntegrityError) { - throw new EteSync.EncryptionPasswordError(error.message); - } else { - throw error; - } - } - - return journals.reduce( - (ret, journal) => { - const journalEntries = entries.get(journal.uid); - let prevUid: string | null = null; - - if (!journalEntries) { - return ret; - } - - const keyPair = userInfo.getKeyPair(userInfoCryptoManager); - const cryptoManager = journal.getCryptoManager(derived, keyPair); - - const collectionInfo = journal.getInfo(cryptoManager); - - const syncEntries = journalEntries.map((entry: EteSync.Entry) => { - const syncEntry = entry.getSyncEntry(cryptoManager, prevUid); - prevUid = entry.uid; - - return syncEntry; - }); - - return ret.set(journal.uid, { - entries: syncEntries, - collection: collectionInfo, - journal, - journalEntries, - }); - }, - Map() - ); - } -); - const PimRouter = withRouter(Pim); -// FIXME: this and withRouters are only needed here because of https://github.com/ReactTraining/react-router/issues/5795 -export default withRouter(function SyncGate(props: RouteComponentProps<{}> & PropsType) { - const etesync = props.etesync; +export default function SyncGate() { + const etebase = useCredentials(); const settings = useSelector((state: StoreState) => state.settings); - const fetchCount = useSelector((state: StoreState) => state.fetchCount); const userInfo = useSelector((state: StoreState) => state.cache.userInfo); const journals = useSelector((state: StoreState) => state.cache.journals); - const entries = useSelector((state: StoreState) => state.cache.entries); + const dispatch = useDispatch(); + const [loading, setLoading] = React.useState(true); + + // Doing this so we refresh on route changes + useHistory(); React.useEffect(() => { - const me = etesync.credentials.email; - const syncAll = () => { - store.dispatch(fetchAll(etesync, entries)).then((haveJournals: boolean) => { - if (haveJournals) { - return; - } - - [ - { - type: "ADDRESS_BOOK", - name: "My Contacts", - }, - { - type: "CALENDAR", - name: "My Calendar", - }, - { - type: "TASKS", - name: "My Tasks", - }, - ].forEach((collectionDesc) => { - const collection = new EteSync.CollectionInfo(); - collection.uid = EteSync.genUid(); - collection.type = collectionDesc.type; - collection.displayName = collectionDesc.name; - - const journal = new EteSync.Journal({ uid: collection.uid }); - const cryptoManager = new EteSync.CryptoManager(etesync.encryptionKey, collection.uid); - journal.setInfo(cryptoManager, collection); - (async () => { - try { - const addedJournalAction = addJournal(etesync, journal); - await addedJournalAction.payload; - store.dispatch(addedJournalAction); - store.dispatch(fetchEntries(etesync, collection.uid)); - } catch (e) { - // FIXME: Limit based on error code to only ignore for associates - console.warn(`Failed creating journal for ${collection.type}. Associate?`); - } - })(); - }); - }); - }; - - if (userInfo) { - syncAll(); - } else { - const fetching = fetchUserInfo(etesync, me); - fetching.payload?.then(() => { - store.dispatch(fetching); - syncAll(); - }).catch(() => { - const userInfo = new EteSync.UserInfo(me, CURRENT_VERSION); - const keyPair = EteSync.AsymmetricCryptoManager.generateKeyPair(); - const cryptoManager = userInfo.getCryptoManager(etesync.encryptionKey); - - userInfo.setKeyPair(cryptoManager, keyPair); - - store.dispatch(createUserInfo(etesync, userInfo)).then(syncAll); - }); - } + (async () => { + const syncManager = SyncManager.getManager(etebase!); + const sync = syncManager.sync(); + dispatch(performSync(sync)); + await sync; + setLoading(false); + })(); }, []); - const entryArrays = entries; - - if ((userInfo === null) || (journals === null) || - ((fetchCount > 0) && - ((entryArrays.size === 0) || entryArrays.some((x) => (x.size === 0)))) - ) { + if (loading) { return (); } // FIXME: Shouldn't be here moment.locale(settings.locale); - - const journalMap = syncInfoSelector({ etesync, userInfo, journals, entries }); + const etesync = etebase as any; return ( @@ -197,33 +78,37 @@ export default withRouter(function SyncGate(props: RouteComponentProps<{}> & Pro )} /> - ( - <> - - - - )} - /> - ( - + ( + <> + + + + )} /> - )} - /> + ( + + )} + /> + + )} & Pro /> ); -}); +} diff --git a/src/sync/SyncManager.ts b/src/sync/SyncManager.ts new file mode 100644 index 0000000..e13e296 --- /dev/null +++ b/src/sync/SyncManager.ts @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: © 2019 EteSync Authors +// SPDX-License-Identifier: GPL-3.0-only + +import * as Etebase from "etebase"; + +import { store, persistor, StoreState } from "../store"; + +import { credentialsSelector } from "../credentials"; +import { setSyncCollection, setSyncGeneral, setCacheItem, setCacheCollection, unsetCacheCollection, unsetCacheItem } from "../store/actions"; + +const cachedSyncManager = new Map(); +export class SyncManager { + private COLLECTION_TYPES = ["etebase.vcard", "etebase.vevent", "etebase.vtodo"]; + private BATCH_SIZE = 40; + + public static getManager(etebase: Etebase.Account) { + const cached = cachedSyncManager.get(etebase.user.username); + if (cached) { + return cached; + } + + const ret = new SyncManager(); + cachedSyncManager.set(etebase.user.username, ret); + return ret; + } + + public static removeManager(etebase: Etebase.Account) { + cachedSyncManager.delete(etebase.user.username); + } + + protected etebase: Etebase.Account; + protected isSyncing: boolean; + + public async fetchCollection(col: Etebase.Collection) { + const storeState = store.getState() as unknown as StoreState; + const etebase = (await credentialsSelector(storeState))!; + const syncCollection = storeState.sync.collections.get(col.uid, undefined); + + const colMgr = etebase.getCollectionManager(); + const itemMgr = colMgr.getItemManager(col); + + let stoken = syncCollection?.stoken; + const limit = this.BATCH_SIZE; + let done = false; + while (!done) { + const items = await itemMgr.list({ stoken, limit }); + for (const item of items.data) { + if (item.isDeleted) { + store.dispatch(unsetCacheItem(col.uid, itemMgr, item.uid)); + } else { + store.dispatch(setCacheItem(col, itemMgr, item)); + } + } + done = items.done; + stoken = items.stoken; + } + + if (syncCollection?.stoken !== stoken) { + store.dispatch(setSyncCollection(col.uid, stoken!)); + } + } + + public async fetchAllCollections() { + const storeState = store.getState() as unknown as StoreState; + const etebase = (await credentialsSelector(storeState))!; + const syncGeneral = storeState.sync.general; + + const colMgr = etebase.getCollectionManager(); + const limit = this.BATCH_SIZE; + let stoken = syncGeneral?.stoken; + let done = false; + while (!done) { + const collections = await colMgr.list({ stoken, limit }); + for (const col of collections.data) { + const meta = await col.getMeta(); + if (this.COLLECTION_TYPES.includes(meta.type)) { + // We only get the changed collections here, so always fetch + if (col.isDeleted) { + store.dispatch(unsetCacheCollection(colMgr, col.uid)); + } else { + store.dispatch(setCacheCollection(colMgr, col)); + } + await this.fetchCollection(col); + } + } + if (collections.removedMemberships) { + for (const removed of collections.removedMemberships) { + store.dispatch(unsetCacheCollection(colMgr, removed.uid)); + } + } + done = collections.done; + stoken = collections.stoken; + } + + if (syncGeneral?.stoken !== stoken) { + store.dispatch(setSyncGeneral(stoken)); + } + return true; + } + + public async sync() { + if (this.isSyncing) { + return false; + } + this.isSyncing = true; + + try { + const stoken = await this.fetchAllCollections(); + return stoken; + } catch (e) { + if (e instanceof Etebase.NetworkError) { + // Ignore network errors + return null; + } else if (e instanceof Etebase.HTTPError) { + switch (e.status) { + case 401: // INVALID TOKEN + case 403: // FORBIDDEN + case 503: // UNAVAILABLE + // FIXME store.dispatch(addNonFatalError(this.etebase, e)); + return null; + } + } + throw e; + } finally { + this.isSyncing = false; + + // Force flusing the store to disk + persistor.persist(); + } + } +}