basic sync

src/LoginGate.tsx

@@ -16,6 +16,7 @@ import * as C from "./constants";
 
 import SignedPagesBadge from "./images/signed-pages-badge.svg";
 import { useCredentials } from "./credentials";
+import LoadingIndicator from "./widgets/LoadingIndicator";
 
 
 export default function LoginGate() {
@@ -35,9 +36,11 @@ export default function LoginGate() {
     }
   }
 
-  const loading = credentials === undefined;
+  if (credentials === undefined) {
-
+    return (
-  if (!credentials) {
+      <LoadingIndicator />
+    );
+  } else if (credentials === null) {
     const style = {
       isSafe: {
         textDecoration: "none",
@@ -55,7 +58,6 @@ export default function LoginGate() {
         <LoginForm
           onSubmit={onFormSubmit}
           error={fetchError}
-          loading={loading}
         />
         <hr style={style.divider} />
         <ExternalLink style={style.isSafe} href="https://www.etesync.com/faq/#signed-pages">
@@ -75,6 +77,6 @@ export default function LoginGate() {
   }
 
   return (
-    <SyncGate etesync={credentials as any} />
+    <SyncGate />
   );
 }

src/SyncGate.tsx

@@ -2,14 +2,13 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import * as React from "react";
-import { useSelector } from "react-redux";
+import { useSelector, useDispatch } from "react-redux";
-import { Route, Switch, Redirect, RouteComponentProps, withRouter } from "react-router";
+import { Route, Switch, Redirect, withRouter, useHistory } from "react-router";
 
 import moment from "moment";
 import "moment/locale/en-gb";
 
 import { List, Map } from "immutable";
-import { createSelector } from "reselect";
 
 import { routeResolver } from "./App";
 
@@ -22,10 +21,12 @@ import Debug from "./Debug";
 import Pim from "./Pim";
 
 import * as EteSync from "etesync";
-import { CURRENT_VERSION } from "etesync";
 
-import { store, JournalsData, EntriesData, StoreState, CredentialsData, UserInfoData } from "./store";
+import { SyncManager } from "./sync/SyncManager";
-import { addJournal, fetchAll, fetchEntries, fetchUserInfo, createUserInfo } from "./store/actions";
+
+import { StoreState } from "./store";
+import { performSync } from "./store/actions";
+import { useCredentials } from "./credentials";
 
 export interface SyncInfoJournal {
   journal: EteSync.Journal;
@@ -36,157 +37,37 @@ export interface SyncInfoJournal {
 
 export type SyncInfo = Map<string, SyncInfoJournal>;
 
-interface PropsType {
-  etesync: CredentialsData;
-}
-
-interface SelectorProps {
-  etesync: CredentialsData;
-  journals: JournalsData;
-  entries: EntriesData;
-  userInfo: UserInfoData;
-}
-
-const syncInfoSelector = createSelector(
-  (props: SelectorProps) => props.etesync,
-  (props: SelectorProps) => props.journals!,
-  (props: SelectorProps) => props.entries,
-  (props: SelectorProps) => props.userInfo,
-  (etesync, journals, entries, userInfo) => {
-    const derived = etesync.encryptionKey;
-    const userInfoCryptoManager = userInfo.getCryptoManager(etesync.encryptionKey);
-    try {
-      userInfo.verify(userInfoCryptoManager);
-    } catch (error) {
-      if (error instanceof EteSync.IntegrityError) {
-        throw new EteSync.EncryptionPasswordError(error.message);
-      } else {
-        throw error;
-      }
-    }
-
-    return journals.reduce(
-      (ret, journal) => {
-        const journalEntries = entries.get(journal.uid);
-        let prevUid: string | null = null;
-
-        if (!journalEntries) {
-          return ret;
-        }
-
-        const keyPair = userInfo.getKeyPair(userInfoCryptoManager);
-        const cryptoManager = journal.getCryptoManager(derived, keyPair);
-
-        const collectionInfo = journal.getInfo(cryptoManager);
-
-        const syncEntries = journalEntries.map((entry: EteSync.Entry) => {
-          const syncEntry = entry.getSyncEntry(cryptoManager, prevUid);
-          prevUid = entry.uid;
-
-          return syncEntry;
-        });
-
-        return ret.set(journal.uid, {
-          entries: syncEntries,
-          collection: collectionInfo,
-          journal,
-          journalEntries,
-        });
-      },
-      Map<string, SyncInfoJournal>()
-    );
-  }
-);
-
 const PimRouter = withRouter(Pim);
 
-// FIXME: this and withRouters are only needed here because of https://github.com/ReactTraining/react-router/issues/5795
+export default function SyncGate() {
-export default withRouter(function SyncGate(props: RouteComponentProps<{}> & PropsType) {
+  const etebase = useCredentials();
-  const etesync = props.etesync;
   const settings = useSelector((state: StoreState) => state.settings);
-  const fetchCount = useSelector((state: StoreState) => state.fetchCount);
   const userInfo = useSelector((state: StoreState) => state.cache.userInfo);
   const journals = useSelector((state: StoreState) => state.cache.journals);
-  const entries = useSelector((state: StoreState) => state.cache.entries);
+  const dispatch = useDispatch();
+  const [loading, setLoading] = React.useState(true);
+
+  // Doing this so we refresh on route changes
+  useHistory();
 
   React.useEffect(() => {
-    const me = etesync.credentials.email;
+    (async () => {
-    const syncAll = () => {
+      const syncManager = SyncManager.getManager(etebase!);
-      store.dispatch<any>(fetchAll(etesync, entries)).then((haveJournals: boolean) => {
+      const sync = syncManager.sync();
-        if (haveJournals) {
+      dispatch(performSync(sync));
-          return;
+      await sync;
-        }
+      setLoading(false);
-
+    })();
-        [
-          {
-            type: "ADDRESS_BOOK",
-            name: "My Contacts",
-          },
-          {
-            type: "CALENDAR",
-            name: "My Calendar",
-          },
-          {
-            type: "TASKS",
-            name: "My Tasks",
-          },
-        ].forEach((collectionDesc) => {
-          const collection = new EteSync.CollectionInfo();
-          collection.uid = EteSync.genUid();
-          collection.type = collectionDesc.type;
-          collection.displayName = collectionDesc.name;
-
-          const journal = new EteSync.Journal({ uid: collection.uid });
-          const cryptoManager = new EteSync.CryptoManager(etesync.encryptionKey, collection.uid);
-          journal.setInfo(cryptoManager, collection);
-          (async () => {
-            try {
-              const addedJournalAction = addJournal(etesync, journal);
-              await addedJournalAction.payload;
-              store.dispatch(addedJournalAction);
-              store.dispatch(fetchEntries(etesync, collection.uid));
-            } catch (e) {
-              // FIXME: Limit based on error code to only ignore for associates
-              console.warn(`Failed creating journal for ${collection.type}. Associate?`);
-            }
-          })();
-        });
-      });
-    };
-
-    if (userInfo) {
-      syncAll();
-    } else {
-      const fetching = fetchUserInfo(etesync, me);
-      fetching.payload?.then(() => {
-        store.dispatch(fetching);
-        syncAll();
-      }).catch(() => {
-        const userInfo = new EteSync.UserInfo(me, CURRENT_VERSION);
-        const keyPair = EteSync.AsymmetricCryptoManager.generateKeyPair();
-        const cryptoManager = userInfo.getCryptoManager(etesync.encryptionKey);
-
-        userInfo.setKeyPair(cryptoManager, keyPair);
-
-        store.dispatch<any>(createUserInfo(etesync, userInfo)).then(syncAll);
-      });
-    }
   }, []);
 
-  const entryArrays = entries;
+  if (loading) {
-
-  if ((userInfo === null) || (journals === null) ||
-    ((fetchCount > 0) &&
-      ((entryArrays.size === 0) || entryArrays.some((x) => (x.size === 0))))
-  ) {
     return (<LoadingIndicator style={{ display: "block", margin: "40px auto" }} />);
   }
 
   // FIXME: Shouldn't be here
   moment.locale(settings.locale);
 
-
+  const etesync = etebase as any;
-  const journalMap = syncInfoSelector({ etesync, userInfo, journals, entries });
 
   return (
     <Switch>
@@ -197,33 +78,37 @@ export default withRouter(function SyncGate(props: RouteComponentProps<{}> & Pro
           <Redirect to={routeResolver.getRoute("pim")} />
         )}
       />
-      <Route
+      {false && (
-        path={routeResolver.getRoute("pim")}
+        <>
-        render={({ history }) => (
+          <Route
-          <>
+            path={routeResolver.getRoute("pim")}
-            <AppBarOverride title="EteSync" />
+            render={({ history }) => (
-            <PimRouter
+              <>
-              etesync={etesync}
+                <AppBarOverride title="EteSync" />
-              userInfo={userInfo}
+                <PimRouter
-              syncInfo={journalMap}
+                  etesync={etesync}
-              history={history}
+                  userInfo={userInfo}
-            />
+                  syncInfo={false as any}
-          </>
+                  history={history}
-        )}
+                />
-      />
+              </>
-      <Route
+            )}
-        path={routeResolver.getRoute("journals")}
-        render={({ location, history }) => (
-          <Journals
-            etesync={etesync}
-            userInfo={userInfo}
-            syncInfo={journalMap}
-            journals={journals}
-            location={location}
-            history={history}
           />
-        )}
+          <Route
-      />
+            path={routeResolver.getRoute("journals")}
+            render={({ location, history }) => (
+              <Journals
+                etesync={etesync}
+                userInfo={userInfo}
+                syncInfo={false as any}
+                journals={journals}
+                location={location}
+                history={history}
+              />
+            )}
+          />
+        </>
+      )}
       <Route
         path={routeResolver.getRoute("settings")}
         exact
@@ -243,4 +128,4 @@ export default withRouter(function SyncGate(props: RouteComponentProps<{}> & Pro
       />
     </Switch>
   );
-});
+}

src/sync/SyncManager.ts

@@ -0,0 +1,131 @@
+// SPDX-FileCopyrightText: © 2019 EteSync Authors
+// SPDX-License-Identifier: GPL-3.0-only
+
+import * as Etebase from "etebase";
+
+import { store, persistor, StoreState } from "../store";
+
+import { credentialsSelector } from "../credentials";
+import { setSyncCollection, setSyncGeneral, setCacheItem, setCacheCollection, unsetCacheCollection, unsetCacheItem } from "../store/actions";
+
+const cachedSyncManager = new Map<string, SyncManager>();
+export class SyncManager {
+  private COLLECTION_TYPES = ["etebase.vcard", "etebase.vevent", "etebase.vtodo"];
+  private BATCH_SIZE = 40;
+
+  public static getManager(etebase: Etebase.Account) {
+    const cached = cachedSyncManager.get(etebase.user.username);
+    if (cached) {
+      return cached;
+    }
+
+    const ret = new SyncManager();
+    cachedSyncManager.set(etebase.user.username, ret);
+    return ret;
+  }
+
+  public static removeManager(etebase: Etebase.Account) {
+    cachedSyncManager.delete(etebase.user.username);
+  }
+
+  protected etebase: Etebase.Account;
+  protected isSyncing: boolean;
+
+  public async fetchCollection(col: Etebase.Collection) {
+    const storeState = store.getState() as unknown as StoreState;
+    const etebase = (await credentialsSelector(storeState))!;
+    const syncCollection = storeState.sync.collections.get(col.uid, undefined);
+
+    const colMgr = etebase.getCollectionManager();
+    const itemMgr = colMgr.getItemManager(col);
+
+    let stoken = syncCollection?.stoken;
+    const limit = this.BATCH_SIZE;
+    let done = false;
+    while (!done) {
+      const items = await itemMgr.list({ stoken, limit });
+      for (const item of items.data) {
+        if (item.isDeleted) {
+          store.dispatch(unsetCacheItem(col.uid, itemMgr, item.uid));
+        } else {
+          store.dispatch(setCacheItem(col, itemMgr, item));
+        }
+      }
+      done = items.done;
+      stoken = items.stoken;
+    }
+
+    if (syncCollection?.stoken !== stoken) {
+      store.dispatch(setSyncCollection(col.uid, stoken!));
+    }
+  }
+
+  public async fetchAllCollections() {
+    const storeState = store.getState() as unknown as StoreState;
+    const etebase = (await credentialsSelector(storeState))!;
+    const syncGeneral = storeState.sync.general;
+
+    const colMgr = etebase.getCollectionManager();
+    const limit = this.BATCH_SIZE;
+    let stoken = syncGeneral?.stoken;
+    let done = false;
+    while (!done) {
+      const collections = await colMgr.list({ stoken, limit });
+      for (const col of collections.data) {
+        const meta = await col.getMeta();
+        if (this.COLLECTION_TYPES.includes(meta.type)) {
+          // We only get the changed collections here, so always fetch
+          if (col.isDeleted) {
+            store.dispatch(unsetCacheCollection(colMgr, col.uid));
+          } else {
+            store.dispatch(setCacheCollection(colMgr, col));
+          }
+          await this.fetchCollection(col);
+        }
+      }
+      if (collections.removedMemberships) {
+        for (const removed of collections.removedMemberships) {
+          store.dispatch(unsetCacheCollection(colMgr, removed.uid));
+        }
+      }
+      done = collections.done;
+      stoken = collections.stoken;
+    }
+
+    if (syncGeneral?.stoken !== stoken) {
+      store.dispatch(setSyncGeneral(stoken));
+    }
+    return true;
+  }
+
+  public async sync() {
+    if (this.isSyncing) {
+      return false;
+    }
+    this.isSyncing = true;
+
+    try {
+      const stoken = await this.fetchAllCollections();
+      return stoken;
+    } catch (e) {
+      if (e instanceof Etebase.NetworkError) {
+        // Ignore network errors
+        return null;
+      } else if (e instanceof Etebase.HTTPError) {
+        switch (e.status) {
+          case 401: // INVALID TOKEN
+          case 403: // FORBIDDEN
+          case 503: // UNAVAILABLE
+            // FIXME store.dispatch(addNonFatalError(this.etebase, e));
+            return null;
+        }
+      }
+      throw e;
+    } finally {
+      this.isSyncing = false;
+
+      // Force flusing the store to disk
+      persistor.persist();
+    }
+  }
+}