From 344bf133cd7c0dac999ee161b5fa36300bb1efb6 Mon Sep 17 00:00:00 2001 From: Christopher Loverich <1010084+cloverich@users.noreply.github.com> Date: Sun, 8 Dec 2024 14:11:43 -0800 Subject: [PATCH] sync on startup; add sync throttle; refactor types - run sync on startup (naive way to avoid desync) - add basic throttling mechanism and stats to sync (view new sync table) - refactor: move types out of preload/documents, to avoid accidental node type inclusion in prod (breaks build) - refactor: use useJournals insstead of useContext(JournalStoreContext) in several places --- src/container.tsx | 6 +- src/electron/migrations/20211005142122.sql | 9 ++ ...rnalsLoader.ts => useApplicationLoader.ts} | 14 ++- src/hooks/useClient.ts | 7 +- src/hooks/useJournals.ts | 2 +- src/preload/client/documents.ts | 108 ++---------------- src/preload/client/importer.ts | 2 +- src/preload/client/index.ts | 2 +- src/preload/client/sync.ts | 36 +++++- src/preload/client/types.ts | 96 ++++++++++++++++ src/views/create/index.tsx | 4 +- src/views/documents/SearchProvider.tsx | 6 +- src/views/documents/index.tsx | 6 +- src/views/documents/sidebar/JournalItem.tsx | 6 +- src/views/documents/sidebar/store.tsx | 6 +- src/views/edit/EditableDocument.ts | 2 +- src/views/edit/PlateContainer.tsx | 6 +- src/views/edit/index.tsx | 6 +- src/views/preferences/index.tsx | 4 +- 19 files changed, 193 insertions(+), 135 deletions(-) rename src/hooks/{useJournalsLoader.ts => useApplicationLoader.ts} (75%) diff --git a/src/container.tsx b/src/container.tsx index 8881d5a..d9052ba 100644 --- a/src/container.tsx +++ b/src/container.tsx @@ -5,8 +5,8 @@ import "react-day-picker/dist/style.css"; import { Navigate, Route, Routes } from "react-router-dom"; import { JournalsStoreContext, - useJournalsLoader, -} from "./hooks/useJournalsLoader"; + useAppLoader, +} from "./hooks/useApplicationLoader"; import Layout, { LayoutDummy } from "./layout"; import DocumentCreator from "./views/create"; import Documents from "./views/documents"; @@ -15,7 +15,7 @@ import Editor from "./views/edit"; import Preferences from "./views/preferences"; export default observer(function Container() { - const { journalsStore, loading, loadingErr } = useJournalsLoader(); + const { journalsStore, loading, loadingErr } = useAppLoader(); if (loading) { return ( diff --git a/src/electron/migrations/20211005142122.sql b/src/electron/migrations/20211005142122.sql index 3aa2015..22fa2a6 100644 --- a/src/electron/migrations/20211005142122.sql +++ b/src/electron/migrations/20211005142122.sql @@ -49,6 +49,15 @@ CREATE TABLE IF NOT EXISTS "document_links" ( PRIMARY KEY ("documentId", "targetId") ); +CREATE TABLE IF NOT EXISTS "sync" ( + "id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + "startedAt" TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + "completedAt" TEXT, + "syncedCount" INTEGER, + "errorCount" INTEGER, + "durationMs" INTEGER +); + CREATE INDEX IF NOT EXISTS "document_links_target_idx" ON "document_links"("targetId"); CREATE INDEX IF NOT EXISTS "documents_title_idx" ON "documents"("title"); CREATE INDEX IF NOT EXISTS "documents_createdat_idx" ON "documents"("createdAt"); diff --git a/src/hooks/useJournalsLoader.ts b/src/hooks/useApplicationLoader.ts similarity index 75% rename from src/hooks/useJournalsLoader.ts rename to src/hooks/useApplicationLoader.ts index 4bcd772..7cec559 100644 --- a/src/hooks/useJournalsLoader.ts +++ b/src/hooks/useApplicationLoader.ts @@ -7,9 +7,10 @@ export const JournalsStoreContext = React.createContext( ); /** - * Loads the journal store. After loading it should be passed down in context + * Runs sync and loads the journal store. After loading it should be passed down in context. + * Could put other application loading state here. */ -export function useJournalsLoader() { +export function useAppLoader() { const [journals, setJournals] = React.useState(); const [journalsStore, setJournalsStore] = React.useState(); const [loading, setLoading] = React.useState(true); @@ -21,6 +22,15 @@ export function useJournalsLoader() { setLoading(true); async function load() { + try { + await client.sync.sync(); + } catch (err: any) { + console.error("error syncing at startup", err); + setLoadingErr(err); + setLoading(false); + return; + } + try { const journalStore = await JournalsStore.init(client); if (!isEffectMounted) return; diff --git a/src/hooks/useClient.ts b/src/hooks/useClient.ts index ce15bcc..5a6c2a4 100644 --- a/src/hooks/useClient.ts +++ b/src/hooks/useClient.ts @@ -1,8 +1,11 @@ import React, { useContext } from "react"; import { IClient } from "../preload/client/types"; -export { SearchResponse } from "../preload/client/documents"; -export { IClient, JournalResponse } from "../preload/client/types"; +export { + IClient, + JournalResponse, + SearchResponse, +} from "../preload/client/types"; export const ClientContext = React.createContext( (window as any).chronicles.createClient(), diff --git a/src/hooks/useJournals.ts b/src/hooks/useJournals.ts index b69de53..81ab479 100644 --- a/src/hooks/useJournals.ts +++ b/src/hooks/useJournals.ts @@ -1,5 +1,5 @@ import React from "react"; -import { JournalsStoreContext } from "./useJournalsLoader"; +import { JournalsStoreContext } from "./useApplicationLoader"; /** * Simple hepler for accessing the journals store diff --git a/src/preload/client/documents.ts b/src/preload/client/documents.ts index af78409..766521d 100644 --- a/src/preload/client/documents.ts +++ b/src/preload/client/documents.ts @@ -1,5 +1,6 @@ import { Database } from "better-sqlite3"; import { Knex } from "knex"; +import path from "path"; import { uuidv7obj } from "uuidv7"; import { mdastToString, parseMarkdown, selectNoteLinks } from "../../markdown"; import { parseNoteLink } from "../../views/edit/editor/features/note-linking/toMdast"; @@ -7,17 +8,14 @@ import { Files } from "../files"; import { IFilesClient } from "./files"; import { parseChroniclesFrontMatter } from "./importer/frontmatter"; import { IPreferencesClient } from "./preferences"; -const path = require("path"); - -export interface GetDocumentResponse { - id: string; - createdAt: string; - updatedAt: string; - title?: string; - content: string; - journal: string; - tags: string[]; -} + +import { + GetDocumentResponse, + SaveRequest, + SearchItem, + SearchRequest, + SearchResponse, +} from "./types"; // table structure of document_links interface DocumentLinkDb { @@ -27,92 +25,6 @@ interface DocumentLinkDb { resolvedAt: string; // todo: unused } -/** - * Structure for searching journal content. - */ -export interface SearchRequest { - /** - * Filter by journal (array of Ids). - * The empty array is treated as "all journals", - * rather than None. - */ - journals: string[]; - - /** - * Filter to documents matching one of these titles - */ - titles?: string[]; - - /** - * Filter documents to those older than a given date - */ - before?: string; - - /** - * Search document body text - */ - texts?: string[]; - - /** - * Search document #tags. ex: ['mytag', 'othertag'] - */ - tags?: string[]; - - limit?: number; - - nodeMatch?: { - /** - * Type of node - * - * https://github.com/syntax-tree/mdast#nodes - */ - type: string; // type of Node - /** - * Match one or more attributes of a node - */ - attributes?: Record; - text?: string; // match raw text from within the node - }; -} - -export type SearchResponse = { - data: SearchItem[]; -}; - -export interface SearchItem { - id: string; - createdAt: string; - title?: string; - journal: string; -} - -export interface SaveRawRequest { - journalName: string; - date: string; - raw: string; -} - -export interface SaveMdastRequest { - journalName: string; - date: string; - mdast: any; -} - -// export type SaveRequest = SaveRawRequest | SaveMdastRequest; - -export interface SaveRequest { - id?: string; - journal: string; - content: string; - title?: string; - tags: string[]; - - // these included for override, originally, - // to support the import process - createdAt?: string; - updatedAt?: string; -} - export type IDocumentsClient = DocumentsClient; export class DocumentsClient { @@ -133,7 +45,7 @@ export class DocumentsClient { .map((row) => row.tag); const filepath = path.join( - this.preferences.get("NOTES_DIR"), + await this.preferences.get("NOTES_DIR"), document.journal, `${id}.md`, ); diff --git a/src/preload/client/importer.ts b/src/preload/client/importer.ts index 4c06c5d..5d22fe4 100644 --- a/src/preload/client/importer.ts +++ b/src/preload/client/importer.ts @@ -366,7 +366,7 @@ export class ImporterClient { console.log("import complete; calling sync to update indexes"); - await this.syncs.sync(); + await this.syncs.sync(true); }; // probably shouldn't make it to final version diff --git a/src/preload/client/index.ts b/src/preload/client/index.ts index dfdbe3d..0aabc77 100644 --- a/src/preload/client/index.ts +++ b/src/preload/client/index.ts @@ -29,7 +29,7 @@ const knex = Knex({ }, }); -export { GetDocumentResponse } from "./documents"; +export { GetDocumentResponse } from "./types"; let client: IClient; diff --git a/src/preload/client/sync.ts b/src/preload/client/sync.ts index f9fcefe..2462390 100644 --- a/src/preload/client/sync.ts +++ b/src/preload/client/sync.ts @@ -3,10 +3,11 @@ import { Knex } from "knex"; import path from "path"; import { UUID } from "uuidv7"; import { Files } from "../files"; -import { GetDocumentResponse, IDocumentsClient } from "./documents"; +import { IDocumentsClient } from "./documents"; import { IFilesClient } from "./files"; import { IJournalsClient } from "./journals"; import { IPreferencesClient } from "./preferences"; +import { GetDocumentResponse } from "./types"; export type ISyncClient = SyncClient; @@ -39,12 +40,27 @@ updatedAt: ${document.updatedAt} /** * Sync the notes directory with the database */ - sync = async () => { + sync = async (force = false) => { + // Skip sync if completed recently; not much thought put into this + const lastSync = await this.knex("sync").orderBy("id", "desc").first(); + if (lastSync?.completedAt && !force) { + const lastSyncDate = new Date(lastSync.completedAt); + const now = new Date(); + const diff = now.getTime() - lastSyncDate.getTime(); + const diffHours = diff / (1000 * 60 * 60); + console.log(`last sync was ${diffHours} ago`); + if (diffHours < 1) { + console.log("skipping sync; last sync was less than an hour ago"); + return; + } + } + + const id = (await this.knex("sync").returning("id").insert({}))[0]; + const start = performance.now(); + this.db.exec("delete from document_tags"); this.db.exec("delete from documents"); this.db.exec("delete from journals"); - // should be automatic, from documents delete cascade - this.db.exec("delete from document_links"); const rootDir = await this.preferences.get("NOTES_DIR"); @@ -62,6 +78,8 @@ updatedAt: ${document.updatedAt} const journals: Record = {}; const erroredDocumentPaths: string[] = []; + let syncedCount = 0; + for await (const file of Files.walk(rootDir, () => true, { // depth: dont go into subdirectories depthLimit: 1, @@ -135,6 +153,7 @@ updatedAt: ${document.updatedAt} createdAt: frontMatter.createdAt, updatedAt: frontMatter.updatedAt, }); + syncedCount++; } catch (e) { erroredDocumentPaths.push(file.path); @@ -170,7 +189,14 @@ updatedAt: ${document.updatedAt} await this.preferences.set("ARCHIVED_JOURNALS", archivedJournals); - // todo: track this in a useful way... + const end = performance.now(); + const durationMs = (end - start).toFixed(2); + await this.knex("sync").where("id", id).update({ + completedAt: new Date().toISOString(), + errorCount: erroredDocumentPaths.length, + syncedCount, + durationMs, + }); console.log("Errored documents (during sync)", erroredDocumentPaths); }; } diff --git a/src/preload/client/types.ts b/src/preload/client/types.ts index 0862582..5281ba8 100644 --- a/src/preload/client/types.ts +++ b/src/preload/client/types.ts @@ -31,3 +31,99 @@ export type JournalResponse = { updatedAt: string; archived: boolean; }; + +export interface GetDocumentResponse { + id: string; + createdAt: string; + updatedAt: string; + title?: string; + content: string; + journal: string; + tags: string[]; +} + +/** + * Structure for searching journal content. + */ +export interface SearchRequest { + /** + * Filter by journal (array of Ids). + * The empty array is treated as "all journals", + * rather than None. + */ + journals: string[]; + + /** + * Filter to documents matching one of these titles + */ + titles?: string[]; + + /** + * Filter documents to those older than a given date + */ + before?: string; + + /** + * Search document body text + */ + texts?: string[]; + + /** + * Search document #tags. ex: ['mytag', 'othertag'] + */ + tags?: string[]; + + limit?: number; + + nodeMatch?: { + /** + * Type of node + * + * https://github.com/syntax-tree/mdast#nodes + */ + type: string; // type of Node + + /** + * Match one or more attributes of a node + */ + attributes?: Record; + text?: string; // match raw text from within the node + }; +} + +export type SearchResponse = { + data: SearchItem[]; +}; + +export interface SearchItem { + id: string; + createdAt: string; + title?: string; + journal: string; +} + +export interface SaveRawRequest { + journalName: string; + date: string; + raw: string; +} + +export interface SaveMdastRequest { + journalName: string; + date: string; + mdast: any; +} +// export type SaveRequest = SaveRawRequest | SaveMdastRequest; + +export interface SaveRequest { + id?: string; + journal: string; + content: string; + title?: string; + tags: string[]; + + // these included for override, originally, + // to support the import process + createdAt?: string; + updatedAt?: string; +} diff --git a/src/views/create/index.tsx b/src/views/create/index.tsx index c5dfd3b..983d711 100644 --- a/src/views/create/index.tsx +++ b/src/views/create/index.tsx @@ -4,14 +4,14 @@ import React, { useContext, useEffect, useState } from "react"; import { useNavigate } from "react-router-dom"; import useClient from "../../hooks/useClient"; import { useIsMounted } from "../../hooks/useIsMounted"; -import { JournalsStoreContext } from "../../hooks/useJournalsLoader"; +import { useJournals } from "../../hooks/useJournals"; import { SearchStoreContext } from "../documents/SearchStore"; import { EditLoadingComponent } from "../edit/loading"; // Creates a new document and immediately navigates to it; re-directs back to // /documents if no journals are available function useCreateDocument() { - const journalsStore = useContext(JournalsStoreContext)!; + const journalsStore = useJournals(); // NOTE: Could move this hook but, but it assumes searchStore is defined, and its setup // in the root documents view. So better to keep it here for now. diff --git a/src/views/documents/SearchProvider.tsx b/src/views/documents/SearchProvider.tsx index a90d13d..f490ab3 100644 --- a/src/views/documents/SearchProvider.tsx +++ b/src/views/documents/SearchProvider.tsx @@ -1,13 +1,13 @@ -import React, { useContext, useEffect, useState } from "react"; +import React, { useEffect, useState } from "react"; import { Outlet, useSearchParams } from "react-router-dom"; import useClient from "../../hooks/useClient"; -import { JournalsStoreContext } from "../../hooks/useJournalsLoader"; +import { useJournals } from "../../hooks/useJournals"; import { LayoutDummy } from "../../layout"; import { SearchStore, SearchStoreContext } from "./SearchStore"; // Sets up document search and its context export function SearchProvider() { - const jstore = useContext(JournalsStoreContext); + const jstore = useJournals(); const client = useClient(); const [params, setParams] = useSearchParams(); const [searchStore, setSearchStore] = useState(null); diff --git a/src/views/documents/index.tsx b/src/views/documents/index.tsx index 5872f04..e651207 100644 --- a/src/views/documents/index.tsx +++ b/src/views/documents/index.tsx @@ -1,15 +1,15 @@ import { Heading, Pane, Paragraph } from "evergreen-ui"; import { observer } from "mobx-react-lite"; -import React, { useContext } from "react"; +import React from "react"; import { useNavigate } from "react-router-dom"; -import { JournalsStoreContext } from "../../hooks/useJournalsLoader"; +import { useJournals } from "../../hooks/useJournals"; import { DocumentItem } from "./DocumentItem"; import { Layout } from "./Layout"; import { SearchStore, useSearchStore } from "./SearchStore"; function DocumentsContainer() { - const journalsStore = useContext(JournalsStoreContext)!; + const journalsStore = useJournals(); const searchStore = useSearchStore()!; const navigate = useNavigate(); diff --git a/src/views/documents/sidebar/JournalItem.tsx b/src/views/documents/sidebar/JournalItem.tsx index 1fc1484..d485dee 100644 --- a/src/views/documents/sidebar/JournalItem.tsx +++ b/src/views/documents/sidebar/JournalItem.tsx @@ -3,12 +3,12 @@ import { cn } from "@udecode/cn"; import { toaster } from "evergreen-ui"; import { noop } from "lodash"; import { observer } from "mobx-react-lite"; -import React, { useContext } from "react"; +import React from "react"; import { useHotkeys } from "react-hotkeys-hook"; import { Icons } from "../../../components/icons"; import { JournalResponse } from "../../../hooks/useClient"; import { useIsMounted } from "../../../hooks/useIsMounted"; -import { JournalsStoreContext } from "../../../hooks/useJournalsLoader"; +import { useJournals } from "../../../hooks/useJournals"; import { SidebarStore } from "./store"; export function JournalCreateForm({ done }: { done: () => any }) { @@ -136,7 +136,7 @@ const JournalEditor = observer(function JournalEditor({ }) { const [name, setName] = React.useState(journal.name); const [saving, setSaving] = React.useState(false); - const store = useContext(JournalsStoreContext)!; + const store = useJournals(); const isMounted = useIsMounted(); const onSave = async () => { diff --git a/src/views/documents/sidebar/store.tsx b/src/views/documents/sidebar/store.tsx index 7c44b82..adb76ab 100644 --- a/src/views/documents/sidebar/store.tsx +++ b/src/views/documents/sidebar/store.tsx @@ -1,9 +1,9 @@ import { toaster } from "evergreen-ui"; import { computed, observable } from "mobx"; -import React, { useContext } from "react"; +import React from "react"; import { JournalsStore } from "../../../hooks/stores/journals"; import { JournalResponse } from "../../../hooks/useClient"; -import { JournalsStoreContext } from "../../../hooks/useJournalsLoader"; +import { useJournals } from "../../../hooks/useJournals"; import { SearchStore } from "../SearchStore"; export function useSidebarStore( @@ -11,7 +11,7 @@ export function useSidebarStore( setIsShown: (isShown: boolean) => any, ) { const searchStore = search; - const jstore = useContext(JournalsStoreContext)!; + const jstore = useJournals(); const [sidebarStore, _] = React.useState( () => new SidebarStore(jstore, searchStore, setIsShown), diff --git a/src/views/edit/EditableDocument.ts b/src/views/edit/EditableDocument.ts index 39468a2..3bdc5a4 100644 --- a/src/views/edit/EditableDocument.ts +++ b/src/views/edit/EditableDocument.ts @@ -3,7 +3,7 @@ import { debounce, pick } from "lodash"; import { IReactionDisposer, computed, observable, reaction, toJS } from "mobx"; import { IClient } from "../../hooks/useClient"; import * as SlateCustom from "../../markdown/remark-slate-transformer/transformers/mdast-to-slate"; -import { GetDocumentResponse } from "../../preload/client/documents"; +import { GetDocumentResponse } from "../../preload/client/types"; import { SlateTransformer } from "./SlateTransformer"; function isExistingDocument( diff --git a/src/views/edit/PlateContainer.tsx b/src/views/edit/PlateContainer.tsx index e29bc0f..4171d48 100644 --- a/src/views/edit/PlateContainer.tsx +++ b/src/views/edit/PlateContainer.tsx @@ -11,7 +11,7 @@ import { isSelectionAtBlockStart, } from "@udecode/plate-common"; import { observer } from "mobx-react-lite"; -import React, { useContext } from "react"; +import React from "react"; // import { Node as SNode } from "slate"; import * as SlateCustom from "../../markdown/remark-slate-transformer/transformers/mdast-to-slate"; @@ -104,7 +104,7 @@ import { } from "./editor/plugins/createVideoPlugin"; import useClient, { JournalResponse } from "../../hooks/useClient"; -import { JournalsStoreContext } from "../../hooks/useJournalsLoader"; +import { useJournals } from "../../hooks/useJournals"; import { SearchStore } from "../documents/SearchStore"; import { EditableDocument } from "./EditableDocument"; import { EditorMode } from "./EditorMode"; @@ -121,7 +121,7 @@ export interface Props { export default observer( ({ children, saving, value, setValue }: React.PropsWithChildren) => { - const jstore = useContext(JournalsStoreContext); + const jstore = useJournals(); const client = useClient(); const store = new SearchStore(client, jstore!, () => {}, []); diff --git a/src/views/edit/index.tsx b/src/views/edit/index.tsx index 4c162ee..b561365 100644 --- a/src/views/edit/index.tsx +++ b/src/views/edit/index.tsx @@ -1,9 +1,9 @@ import { ChevronLeftIcon, IconButton, Pane } from "evergreen-ui"; import { observer } from "mobx-react-lite"; -import React, { useContext, useEffect, useState } from "react"; +import React, { useEffect, useState } from "react"; import { useNavigate, useParams } from "react-router-dom"; import { JournalResponse } from "../../hooks/useClient"; -import { JournalsStoreContext } from "../../hooks/useJournalsLoader"; +import { useJournals } from "../../hooks/useJournals"; import Titlebar from "../../titlebar/macos"; import { useSearchStore } from "../documents/SearchStore"; import * as Base from "../layout"; @@ -20,7 +20,7 @@ import { useEditableDocument } from "./useEditableDocument"; // Loads document, with loading and error placeholders const DocumentLoadingContainer = observer(() => { - const journalsStore = useContext(JournalsStoreContext)!; + const journalsStore = useJournals(); const { document: documentId } = useParams(); // todo: handle missing or invalid documentId; loadingError may be fine for this, but diff --git a/src/views/preferences/index.tsx b/src/views/preferences/index.tsx index fad7785..f5b8c84 100644 --- a/src/views/preferences/index.tsx +++ b/src/views/preferences/index.tsx @@ -71,7 +71,9 @@ const Preferences = observer(() => { toaster.notify("Syncing cache...may take a few minutes"); store.loading = true; - await client.sync.sync(); + + // force sync when called manually + await client.sync.sync(true); await jstore.refresh(); store.loading = false; toaster.success("Cache synced");