sync on startup; add sync throttle; refactor types

- run sync on startup (naive way to avoid desync)
- add basic throttling mechanism and stats to sync (view new sync table)
- refactor: move types out of preload/documents, to avoid accidental node type inclusion in prod (breaks build)
- refactor: use useJournals insstead of useContext(JournalStoreContext) in several places

src/container.tsx

@@ -5,8 +5,8 @@ import "react-day-picker/dist/style.css";
 import { Navigate, Route, Routes } from "react-router-dom";
 import {
   JournalsStoreContext,
-  useJournalsLoader,
-} from "./hooks/useJournalsLoader";
+  useAppLoader,
+} from "./hooks/useApplicationLoader";
 import Layout, { LayoutDummy } from "./layout";
 import DocumentCreator from "./views/create";
 import Documents from "./views/documents";
@@ -15,7 +15,7 @@ import Editor from "./views/edit";
 import Preferences from "./views/preferences";
 
 export default observer(function Container() {
-  const { journalsStore, loading, loadingErr } = useJournalsLoader();
+  const { journalsStore, loading, loadingErr } = useAppLoader();
 
   if (loading) {
     return (

src/electron/migrations/20211005142122.sql

@@ -49,6 +49,15 @@ CREATE TABLE IF NOT EXISTS "document_links" (
     PRIMARY KEY ("documentId", "targetId")
 );
 
+CREATE TABLE IF NOT EXISTS "sync" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "startedAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "completedAt" TEXT,
+    "syncedCount" INTEGER,
+    "errorCount" INTEGER,
+    "durationMs" INTEGER
+);
+
 CREATE INDEX IF NOT EXISTS "document_links_target_idx" ON "document_links"("targetId");
 CREATE INDEX IF NOT EXISTS "documents_title_idx" ON "documents"("title");
 CREATE INDEX IF NOT EXISTS "documents_createdat_idx" ON "documents"("createdAt");

src/hooks/useJournalsLoader.ts → src/hooks/useApplicationLoader.ts

@@ -7,9 +7,10 @@ export const JournalsStoreContext = React.createContext<JournalsStore | null>(
 );
 
 /**
- * Loads the journal store. After loading it should be passed down in context
+ * Runs sync and loads the journal store. After loading it should be passed down in context.
+ * Could put other application loading state here.
  */
-export function useJournalsLoader() {
+export function useAppLoader() {
   const [journals, setJournals] = React.useState<JournalResponse[]>();
   const [journalsStore, setJournalsStore] = React.useState<JournalsStore>();
   const [loading, setLoading] = React.useState(true);
@@ -21,6 +22,15 @@ export function useJournalsLoader() {
     setLoading(true);
 
     async function load() {
+      try {
+        await client.sync.sync();
+      } catch (err: any) {
+        console.error("error syncing at startup", err);
+        setLoadingErr(err);
+        setLoading(false);
+        return;
+      }
+
       try {
         const journalStore = await JournalsStore.init(client);
         if (!isEffectMounted) return;

src/hooks/useClient.ts

@@ -1,8 +1,11 @@
 import React, { useContext } from "react";
 import { IClient } from "../preload/client/types";
 
-export { SearchResponse } from "../preload/client/documents";
-export { IClient, JournalResponse } from "../preload/client/types";
+export {
+  IClient,
+  JournalResponse,
+  SearchResponse,
+} from "../preload/client/types";
 
 export const ClientContext = React.createContext<any>(
   (window as any).chronicles.createClient(),

src/hooks/useJournals.ts

@@ -1,5 +1,5 @@
 import React from "react";
-import { JournalsStoreContext } from "./useJournalsLoader";
+import { JournalsStoreContext } from "./useApplicationLoader";
 
 /**
  * Simple hepler for accessing the journals store

src/preload/client/documents.ts

@@ -1,23 +1,21 @@
 import { Database } from "better-sqlite3";
 import { Knex } from "knex";
+import path from "path";
 import { uuidv7obj } from "uuidv7";
 import { mdastToString, parseMarkdown, selectNoteLinks } from "../../markdown";
 import { parseNoteLink } from "../../views/edit/editor/features/note-linking/toMdast";
 import { Files } from "../files";
 import { IFilesClient } from "./files";
 import { parseChroniclesFrontMatter } from "./importer/frontmatter";
 import { IPreferencesClient } from "./preferences";
-const path = require("path");
-
-export interface GetDocumentResponse {
-  id: string;
-  createdAt: string;
-  updatedAt: string;
-  title?: string;
-  content: string;
-  journal: string;
-  tags: string[];
-}
+
+import {
+  GetDocumentResponse,
+  SaveRequest,
+  SearchItem,
+  SearchRequest,
+  SearchResponse,
+} from "./types";
 
 // table structure of document_links
 interface DocumentLinkDb {
@@ -27,92 +25,6 @@ interface DocumentLinkDb {
   resolvedAt: string; // todo: unused
 }
 
-/**
- * Structure for searching journal content.
- */
-export interface SearchRequest {
-  /**
-   * Filter by journal (array of Ids).
-   * The empty array is treated as "all journals",
-   * rather than None.
-   */
-  journals: string[];
-
-  /**
-   * Filter to documents matching one of these titles
-   */
-  titles?: string[];
-
-  /**
-   * Filter documents to those older than a given date
-   */
-  before?: string;
-
-  /**
-   * Search document body text
-   */
-  texts?: string[];
-
-  /**
-   * Search document #tags. ex: ['mytag', 'othertag']
-   */
-  tags?: string[];
-
-  limit?: number;
-
-  nodeMatch?: {
-    /**
-     * Type of node
-     *
-     * https://github.com/syntax-tree/mdast#nodes
-     */
-    type: string; // type of Node
-    /**
-     * Match one or more attributes of a node
-     */
-    attributes?: Record<string, string | number>;
-    text?: string; // match raw text from within the node
-  };
-}
-
-export type SearchResponse = {
-  data: SearchItem[];
-};
-
-export interface SearchItem {
-  id: string;
-  createdAt: string;
-  title?: string;
-  journal: string;
-}
-
-export interface SaveRawRequest {
-  journalName: string;
-  date: string;
-  raw: string;
-}
-
-export interface SaveMdastRequest {
-  journalName: string;
-  date: string;
-  mdast: any;
-}
-
-// export type SaveRequest = SaveRawRequest | SaveMdastRequest;
-
-export interface SaveRequest {
-  id?: string;
-  journal: string;
-  content: string;
-  title?: string;
-  tags: string[];
-
-  // these included for override, originally,
-  // to support the import process
-  createdAt?: string;
-  updatedAt?: string;
-}
-
 export type IDocumentsClient = DocumentsClient;
 
 export class DocumentsClient {
@@ -133,7 +45,7 @@ export class DocumentsClient {
       .map((row) => row.tag);
 
     const filepath = path.join(
-      this.preferences.get("NOTES_DIR"),
+      await this.preferences.get("NOTES_DIR"),
       document.journal,
       `${id}.md`,
     );

src/preload/client/importer.ts

@@ -366,7 +366,7 @@ export class ImporterClient {
 
     console.log("import complete; calling sync to update indexes");
 
-    await this.syncs.sync();
+    await this.syncs.sync(true);
   };
 
   // probably shouldn't make it to final version

src/preload/client/index.ts

@@ -29,7 +29,7 @@ const knex = Knex({
   },
 });
 
-export { GetDocumentResponse } from "./documents";
+export { GetDocumentResponse } from "./types";
 
 let client: IClient;
 

src/preload/client/sync.ts

@@ -3,10 +3,11 @@ import { Knex } from "knex";
 import path from "path";
 import { UUID } from "uuidv7";
 import { Files } from "../files";
-import { GetDocumentResponse, IDocumentsClient } from "./documents";
+import { IDocumentsClient } from "./documents";
 import { IFilesClient } from "./files";
 import { IJournalsClient } from "./journals";
 import { IPreferencesClient } from "./preferences";
+import { GetDocumentResponse } from "./types";
 
 export type ISyncClient = SyncClient;
 
@@ -39,12 +40,27 @@ updatedAt: ${document.updatedAt}
   /**
    * Sync the notes directory with the database
    */
-  sync = async () => {
+  sync = async (force = false) => {
+    // Skip sync if completed recently; not much thought put into this
+    const lastSync = await this.knex("sync").orderBy("id", "desc").first();
+    if (lastSync?.completedAt && !force) {
+      const lastSyncDate = new Date(lastSync.completedAt);
+      const now = new Date();
+      const diff = now.getTime() - lastSyncDate.getTime();
+      const diffHours = diff / (1000 * 60 * 60);
+      console.log(`last sync was ${diffHours} ago`);
+      if (diffHours < 1) {
+        console.log("skipping sync; last sync was less than an hour ago");
+        return;
+      }
+    }
+
+    const id = (await this.knex("sync").returning("id").insert({}))[0];
+    const start = performance.now();
+
     this.db.exec("delete from document_tags");
     this.db.exec("delete from documents");
     this.db.exec("delete from journals");
-    // should be automatic, from documents delete cascade
-    this.db.exec("delete from document_links");
 
     const rootDir = await this.preferences.get("NOTES_DIR");
 
@@ -62,6 +78,8 @@ updatedAt: ${document.updatedAt}
     const journals: Record<string, number> = {};
     const erroredDocumentPaths: string[] = [];
 
+    let syncedCount = 0;
+
     for await (const file of Files.walk(rootDir, () => true, {
       // depth: dont go into subdirectories
       depthLimit: 1,
@@ -135,6 +153,7 @@ updatedAt: ${document.updatedAt}
           createdAt: frontMatter.createdAt,
           updatedAt: frontMatter.updatedAt,
         });
+        syncedCount++;
       } catch (e) {
         erroredDocumentPaths.push(file.path);
 
@@ -170,7 +189,14 @@ updatedAt: ${document.updatedAt}
 
     await this.preferences.set("ARCHIVED_JOURNALS", archivedJournals);
 
-    // todo: track this in a useful way...
+    const end = performance.now();
+    const durationMs = (end - start).toFixed(2);
+    await this.knex("sync").where("id", id).update({
+      completedAt: new Date().toISOString(),
+      errorCount: erroredDocumentPaths.length,
+      syncedCount,
+      durationMs,
+    });
     console.log("Errored documents (during sync)", erroredDocumentPaths);
   };
 }