sync follow up issues

src/container.tsx

@@ -5,8 +5,8 @@ import "react-day-picker/dist/style.css";
 import { Navigate, Route, Routes } from "react-router-dom";
 import {
   JournalsStoreContext,
-  useJournalsLoader,
-} from "./hooks/useJournalsLoader";
+  useAppLoader,
+} from "./hooks/useApplicationLoader";
 import Layout, { LayoutDummy } from "./layout";
 import DocumentCreator from "./views/create";
 import Documents from "./views/documents";
@@ -15,7 +15,7 @@ import Editor from "./views/edit";
 import Preferences from "./views/preferences";
 
 export default observer(function Container() {
-  const { journalsStore, loading, loadingErr } = useJournalsLoader();
+  const { journalsStore, loading, loadingErr } = useAppLoader();
 
   if (loading) {
     return (

src/electron/migrations/20211005142122.sql

@@ -49,6 +49,15 @@ CREATE TABLE IF NOT EXISTS "document_links" (
     PRIMARY KEY ("documentId", "targetId")
 );
 
+CREATE TABLE IF NOT EXISTS "sync" (
+    "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    "startedAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "completedAt" TEXT,
+    "syncedCount" INTEGER,
+    "errorCount" INTEGER,
+    "durationMs" INTEGER
+);
+
 CREATE INDEX IF NOT EXISTS "document_links_target_idx" ON "document_links"("targetId");
 CREATE INDEX IF NOT EXISTS "documents_title_idx" ON "documents"("title");
 CREATE INDEX IF NOT EXISTS "documents_createdat_idx" ON "documents"("createdAt");

src/hooks/useJournalsLoader.ts → src/hooks/useApplicationLoader.ts

@@ -7,9 +7,10 @@ export const JournalsStoreContext = React.createContext<JournalsStore | null>(
 );
 
 /**
- * Loads the journal store. After loading it should be passed down in context
+ * Runs sync and loads the journal store. After loading it should be passed down in context.
+ * Could put other application loading state here.
  */
-export function useJournalsLoader() {
+export function useAppLoader() {
   const [journals, setJournals] = React.useState<JournalResponse[]>();
   const [journalsStore, setJournalsStore] = React.useState<JournalsStore>();
   const [loading, setLoading] = React.useState(true);
@@ -21,6 +22,15 @@ export function useJournalsLoader() {
     setLoading(true);
 
     async function load() {
+      try {
+        await client.sync.sync();
+      } catch (err: any) {
+        console.error("error syncing at startup", err);
+        setLoadingErr(err);
+        setLoading(false);
+        return;
+      }
+
       try {
         const journalStore = await JournalsStore.init(client);
         if (!isEffectMounted) return;

src/hooks/useClient.ts

@@ -1,8 +1,11 @@
 import React, { useContext } from "react";
 import { IClient } from "../preload/client/types";
 
-export { SearchResponse } from "../preload/client/documents";
-export { IClient, JournalResponse } from "../preload/client/types";
+export {
+  IClient,
+  JournalResponse,
+  SearchResponse,
+} from "../preload/client/types";
 
 export const ClientContext = React.createContext<any>(
   (window as any).chronicles.createClient(),

src/hooks/useJournals.ts

@@ -1,5 +1,5 @@
 import React from "react";
-import { JournalsStoreContext } from "./useJournalsLoader";
+import { JournalsStoreContext } from "./useApplicationLoader";
 
 /**
  * Simple hepler for accessing the journals store

src/preload/client/documents.ts

@@ -1,19 +1,21 @@
 import { Database } from "better-sqlite3";
 import { Knex } from "knex";
+import path from "path";
 import { uuidv7obj } from "uuidv7";
 import { mdastToString, parseMarkdown, selectNoteLinks } from "../../markdown";
 import { parseNoteLink } from "../../views/edit/editor/features/note-linking/toMdast";
+import { Files } from "../files";
 import { IFilesClient } from "./files";
+import { parseChroniclesFrontMatter } from "./importer/frontmatter";
+import { IPreferencesClient } from "./preferences";
 
-export interface GetDocumentResponse {
-  id: string;
-  createdAt: string;
-  updatedAt: string;
-  title?: string;
-  content: string;
-  journal: string;
-  tags: string[];
-}
+import {
+  GetDocumentResponse,
+  SaveRequest,
+  SearchItem,
+  SearchRequest,
+  SearchResponse,
+} from "./types";
 
 // table structure of document_links
 interface DocumentLinkDb {
@@ -23,115 +25,54 @@ interface DocumentLinkDb {
   resolvedAt: string; // todo: unused
 }
 
-/**
- * Structure for searching journal content.
- */
-export interface SearchRequest {
-  /**
-   * Filter by journal (array of Ids).
-   * The empty array is treated as "all journals",
-   * rather than None.
-   */
-  journals: string[];
-
-  /**
-   * Filter to documents matching one of these titles
-   */
-  titles?: string[];
-
-  /**
-   * Filter documents to those older than a given date
-   */
-  before?: string;
-
-  /**
-   * Search document body text
-   */
-  texts?: string[];
-
-  /**
-   * Search document #tags. ex: ['mytag', 'othertag']
-   */
-  tags?: string[];
-
-  limit?: number;
-
-  nodeMatch?: {
-    /**
-     * Type of node
-     *
-     * https://github.com/syntax-tree/mdast#nodes
-     */
-    type: string; // type of Node
-    /**
-     * Match one or more attributes of a node
-     */
-    attributes?: Record<string, string | number>;
-    text?: string; // match raw text from within the node
-  };
-}
-
-export type SearchResponse = {
-  data: SearchItem[];
-};
-
-export interface SearchItem {
-  id: string;
-  createdAt: string;
-  title?: string;
-  journal: string;
-}
-
-export interface SaveRawRequest {
-  journalName: string;
-  date: string;
-  raw: string;
-}
-
-export interface SaveMdastRequest {
-  journalName: string;
-  date: string;
-  mdast: any;
-}
-
-// export type SaveRequest = SaveRawRequest | SaveMdastRequest;
-
-export interface SaveRequest {
-  id?: string;
-  journal: string;
-  content: string;
-  title?: string;
-  tags: string[];
-
-  // these included for override, originally,
-  // to support the import process
-  createdAt?: string;
-  updatedAt?: string;
-}
-
 export type IDocumentsClient = DocumentsClient;
 
 export class DocumentsClient {
   constructor(
     private db: Database,
     private knex: Knex,
     private files: IFilesClient,
+    private preferences: IPreferencesClient,
   ) {}
 
-  findById = ({ id }: { id: string }): Promise<GetDocumentResponse> => {
+  findById = async ({ id }: { id: string }): Promise<GetDocumentResponse> => {
     const document = this.db
       .prepare(`SELECT * FROM documents WHERE id = :id`)
       .get({ id });
     const documentTags = this.db
       .prepare(`SELECT tag FROM document_tags WHERE documentId = :documentId`)
       .all({ documentId: id })
       .map((row) => row.tag);
+
+    const filepath = path.join(
+      await this.preferences.get("NOTES_DIR"),
+      document.journal,
+      `${id}.md`,
+    );
+
+    // freshly load the document from disk to avoid desync issues
+    // todo: a real strategy for keeping db in sync w/ filesystem, that allows
+    // loading from db.
+    const { contents } = await this.loadDoc(filepath);
+
     return {
       ...document,
+      contents,
       tags: documentTags,
     };
   };
 
+  // load a document + parse frontmatter from a file
+  loadDoc = async (path: string) => {
+    // todo: validate path is in notes dir
+    // const rootDir = await this.preferences.get("NOTES_DIR");
+    // todo: sha comparison
+    const contents = await Files.read(path);
+    const { frontMatter, body } = parseChroniclesFrontMatter(contents);
+
+    return { contents: body, frontMatter };
+  };
+
   del = async (id: string, journal: string) => {
     await this.files.deleteDocument(id, journal);
     this.db.prepare("delete from documents where id = :id").run({ id });

src/preload/client/importer.ts

@@ -366,7 +366,7 @@ export class ImporterClient {
 
     console.log("import complete; calling sync to update indexes");
 
-    await this.syncs.sync();
+    await this.syncs.sync(true);
   };
 
   // probably shouldn't make it to final version

src/preload/client/importer/frontmatter.ts

@@ -266,3 +266,54 @@ function preprocessRawFrontMatter(content: string) {
       })
   );
 }
+
+function preprocessChroniclesFrontMatter(content: string) {
+  // Regular expression to match key-value pairs in front matter
+  return content
+    .replace(/^(\w+):\s*$/gm, '$1: ""') // Handle keys with no values
+    .replace(/^(\w+):\s*(.+)$/gm, (match, key, value) => {
+      // Check if value contains special characters that need quoting
+      if (value.match(/[:{}[\],&*#?|\-<>=!%@`]/) || value.includes("\n")) {
+        // If the value is not already quoted, wrap it in double quotes
+        if (!/^['"].*['"]$/.test(value)) {
+          // Escape any existing double quotes in the value
+          value = value.replace(/"/g, '\\"');
+          return `${key}: "${value}"`;
+        }
+      }
+      return match; // Return unchanged if no special characters
+    });
+}
+
+// naive frontmatter parser for files formatted in chronicles style...
+// which  just means a regular markdown file + yaml front matter
+// ... todo: use remark ecosystem parser
+export function parseChroniclesFrontMatter(content: string) {
+  // Regular expression to match front matter (--- at the beginning and end)
+  const frontMatterRegex = /^---\n([\s\S]*?)\n---\n*/;
+
+  // Match the front matter
+  const match = content.match(frontMatterRegex);
+  if (!match) {
+    return {
+      frontMatter: {}, // No front matter found
+      body: content, // Original content without changes
+    };
+  }
+
+  // Extract front matter and body
+  const frontMatterContent = preprocessChroniclesFrontMatter(match[1]);
+  const body = content.slice(match[0].length); // Content without front matter
+
+  // Parse the front matter using yaml
+  const frontMatter = yaml.parse(frontMatterContent);
+  frontMatter.tags = frontMatter.tags
+    .split(",")
+    .map((tag: string) => tag.trim())
+    .filter(Boolean);
+
+  return {
+    frontMatter,
+    body,
+  };
+}

src/preload/client/index.ts

@@ -29,7 +29,7 @@ const knex = Knex({
   },
 });
 
-export { GetDocumentResponse } from "./documents";
+export { GetDocumentResponse } from "./types";
 
 let client: IClient;
 
@@ -45,7 +45,7 @@ export function create(): IClient {
     const preferences = new PreferencesClient(settings);
     const files = new FilesClient(settings);
     const journals = new JournalsClient(db, files, preferences);
-    const documents = new DocumentsClient(db, knex, files);
+    const documents = new DocumentsClient(db, knex, files, preferences);
     const sync = new SyncClient(
       db,
       knex,