Skip to content

Commit

Permalink
always load document from disk
Browse files Browse the repository at this point in the history 
- read documents directly from disk, rather than db, to sidestep desync issues
  • Loading branch information
@cloverich
cloverich committed Dec 8, 2024
1 parent 7d16a3a commit 64f08b1
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 80 deletions.
31 changes: 30 additions & 1 deletion src/preload/client/documents.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@ import { Knex } from "knex";
import { uuidv7obj } from "uuidv7";
import { mdastToString, parseMarkdown, selectNoteLinks } from "../../markdown";
import { parseNoteLink } from "../../views/edit/editor/features/note-linking/toMdast";
import { Files } from "../files";
import { IFilesClient } from "./files";
import { parseChroniclesFrontMatter } from "./importer/frontmatter";
import { IPreferencesClient } from "./preferences";
const path = require("path");

export interface GetDocumentResponse {
id: string;
Expand Down Expand Up @@ -116,22 +120,47 @@ export class DocumentsClient {
private db: Database,
private knex: Knex,
private files: IFilesClient,
private preferences: IPreferencesClient,
) {}

findById = ({ id }: { id: string }): Promise<GetDocumentResponse> => {
findById = async ({ id }: { id: string }): Promise<GetDocumentResponse> => {
const document = this.db
.prepare(`SELECT * FROM documents WHERE id = :id`)
.get({ id });
const documentTags = this.db
.prepare(`SELECT tag FROM document_tags WHERE documentId = :documentId`)
.all({ documentId: id })
.map((row) => row.tag);

const filepath = path.join(
this.preferences.get("NOTES_DIR"),
document.journal,
`${id}.md`,
);

// freshly load the document from disk to avoid desync issues
// todo: a real strategy for keeping db in sync w/ filesystem, that allows
// loading from db.
const { contents } = await this.loadDoc(filepath);

return {
...document,
contents,
tags: documentTags,
};
};

// load a document + parse frontmatter from a file
loadDoc = async (path: string) => {
// todo: validate path is in notes dir
// const rootDir = await this.preferences.get("NOTES_DIR");
// todo: sha comparison
const contents = await Files.read(path);
const { frontMatter, body } = parseChroniclesFrontMatter(contents);

return { contents: body, frontMatter };
};

del = async (id: string, journal: string) => {
await this.files.deleteDocument(id, journal);
this.db.prepare("delete from documents where id = :id").run({ id });
Expand Down
51 changes: 51 additions & 0 deletions src/preload/client/importer/frontmatter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,3 +266,54 @@ function preprocessRawFrontMatter(content: string) {
})
);
}

function preprocessChroniclesFrontMatter(content: string) {
// Regular expression to match key-value pairs in front matter
return content
.replace(/^(\w+):\s*$/gm, '$1: ""') // Handle keys with no values
.replace(/^(\w+):\s*(.+)$/gm, (match, key, value) => {
// Check if value contains special characters that need quoting
if (value.match(/[:{}[\],&*#?|\-<>=!%@`]/) || value.includes("\n")) {
// If the value is not already quoted, wrap it in double quotes
if (!/^['"].*['"]$/.test(value)) {
// Escape any existing double quotes in the value
value = value.replace(/"/g, '\\"');
return `${key}: "${value}"`;
}
}
return match; // Return unchanged if no special characters
});
}

// naive frontmatter parser for files formatted in chronicles style...
// which just means a regular markdown file + yaml front matter
// ... todo: use remark ecosystem parser
export function parseChroniclesFrontMatter(content: string) {
// Regular expression to match front matter (--- at the beginning and end)
const frontMatterRegex = /^---\n([\s\S]*?)\n---\n*/;

// Match the front matter
const match = content.match(frontMatterRegex);
if (!match) {
return {
frontMatter: {}, // No front matter found
body: content, // Original content without changes
};
}

// Extract front matter and body
const frontMatterContent = preprocessChroniclesFrontMatter(match[1]);
const body = content.slice(match[0].length); // Content without front matter

// Parse the front matter using yaml
const frontMatter = yaml.parse(frontMatterContent);
frontMatter.tags = frontMatter.tags
.split(",")
.map((tag: string) => tag.trim())
.filter(Boolean);

return {
frontMatter,
body,
};
}
2 changes: 1 addition & 1 deletion src/preload/client/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ export function create(): IClient {
const preferences = new PreferencesClient(settings);
const files = new FilesClient(settings);
const journals = new JournalsClient(db, files, preferences);
const documents = new DocumentsClient(db, knex, files);
const documents = new DocumentsClient(db, knex, files, preferences);
const sync = new SyncClient(
db,
knex,
Expand Down
86 changes: 8 additions & 78 deletions src/preload/client/sync.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import { Database } from "better-sqlite3";
import { Knex } from "knex";
import path from "path";
import { UUID } from "uuidv7";
import yaml from "yaml";
import { Files } from "../files";
import { GetDocumentResponse, IDocumentsClient } from "./documents";
import { IFilesClient } from "./files";
Expand All @@ -11,55 +10,6 @@ import { IPreferencesClient } from "./preferences";

export type ISyncClient = SyncClient;

function preprocessFrontMatter(content: string) {
// Regular expression to match key-value pairs in front matter
return content
.replace(/^(\w+):\s*$/gm, '$1: ""') // Handle keys with no values
.replace(/^(\w+):\s*(.+)$/gm, (match, key, value) => {
// Check if value contains special characters that need quoting
if (value.match(/[:{}[\],&*#?|\-<>=!%@`]/) || value.includes("\n")) {
// If the value is not already quoted, wrap it in double quotes
if (!/^['"].*['"]$/.test(value)) {
// Escape any existing double quotes in the value
value = value.replace(/"/g, '\\"');
return `${key}: "${value}"`;
}
}
return match; // Return unchanged if no special characters
});
}

// naive frontmatter parser
function parseFrontMatter(content: string) {
// Regular expression to match front matter (--- at the beginning and end)
const frontMatterRegex = /^---\n([\s\S]*?)\n---\n*/;

// Match the front matter
const match = content.match(frontMatterRegex);
if (!match) {
return {
frontMatter: {}, // No front matter found
body: content, // Original content without changes
};
}

// Extract front matter and body
const frontMatterContent = preprocessFrontMatter(match[1]);
const body = content.slice(match[0].length); // Content without front matter

// Parse the front matter using yaml
const frontMatter = yaml.parse(frontMatterContent);
frontMatter.tags = frontMatter.tags
.split(",")
.map((tag: string) => tag.trim())
.filter(Boolean);

return {
frontMatter,
body,
};
}

const SKIPPABLE_FILES = new Set(".DS_Store");

export class SyncClient {
Expand All @@ -72,20 +22,6 @@ export class SyncClient {
private preferences: IPreferencesClient,
) {}

// private selectImages = (mdast: any, images: any = new Set()) => {
// if (mdast.type === "image") {
// images.add(mdast.url);
// }

// if (mdast.children) {
// for (const child of mdast.children) {
// this.selectImages(child, images);
// }
// }

// return images;
// };

/**
* Convert the properties we track to frontmatter
*/
Expand Down Expand Up @@ -120,7 +56,6 @@ updatedAt: ${document.updatedAt}
await this.files.ensureDir(path.join(rootDir, "_attachments"));

console.log("syncing directory", rootDir);
const rootFolderName = path.basename(rootDir);

// Track created journals and number of documents to help troubleshoot
// sync issues
Expand Down Expand Up @@ -158,7 +93,12 @@ updatedAt: ${document.updatedAt}
try {
UUID.parse(documentId);
} catch (e) {
console.error("Invalid document id", documentId, e);
console.error(
"Invalid document id in sync; skipping",
file.path,
documentId,
e,
);
continue;
}

Expand All @@ -179,17 +119,7 @@ updatedAt: ${document.updatedAt}
journals[dirname] = 0;
}

// todo: sha comparison
const contents = await Files.read(file.path);
const { frontMatter, body } = parseFrontMatter(contents);

// In a directory that was pre-formatted by Chronicles, this should not
// be needed. Will leave here as a reminder when I do the more generalized
// import routine.
if (!frontMatter.createdAt) {
frontMatter.createdAt = file.stats.ctime.toISOString();
frontMatter.updatedAt = file.stats.mtime.toISOString();
}
const { contents, frontMatter } = await this.documents.loadDoc(file.path);

// todo: handle additional kinds of frontMatter; just add a column for them
// and ensure they are not overwritten when editing existing files
Expand All @@ -199,7 +129,7 @@ updatedAt: ${document.updatedAt}
await this.documents.createIndex({
id: documentId,
journal: dirname, // using name as id
content: body,
content: contents,
title: frontMatter.title,
tags: frontMatter.tags || [],
createdAt: frontMatter.createdAt,
Expand Down

0 comments on commit 64f08b1

Please sign in to comment.