persist and maintain front matter (#283)

import and track front matter; misc minor fixes and refactors

- on import, track front matter in staging table if parsed
- work core Chronicles metadata (tags, title, dates) into front matter
- maintain front matter in documents (keep existing, merge Chronicles properties)
- refactor: migrate rest of client.documents to knex
- refactor: parse and stringify frontmatter tags as valid yaml
- add micromark front matter libraries and prefer them when parsing / serializing
- fix: parse and serialize tags as valid yaml array
- fix: When document not found, display error and fix back button (no-op before)
- fix: Correctly set default journal on sync, and add an additional fallback
- fix: CACHE_DIR / DATABSE_URL references

package.json

@@ -58,8 +58,10 @@
     "lucide-react": "^0.314.0",
     "luxon": "^2.4.0",
     "mdast-util-from-markdown": "^2.0.2",
+    "mdast-util-frontmatter": "^2.0.1",
     "mdast-util-gfm": "^3.0.0",
     "mdast-util-to-markdown": "^2.1.2",
+    "micromark-extension-frontmatter": "^2.0.0",
     "micromark-extension-gfm": "^3.0.0",
     "mobx": "^5.15.4",
     "mobx-react-lite": "^2.0.7",

src/electron/migrations/20211005142122.sql

@@ -26,6 +26,7 @@ CREATE TABLE  IF NOT EXISTS "documents" (
     "title" TEXT,
     "content" TEXT NOT NULL,
     "journal" TEXT NOT NULL,
+    "frontmatter" TEXT NOT NULL,
     FOREIGN KEY ("journal") REFERENCES "journals" ("name") ON DELETE CASCADE ON UPDATE CASCADE
 );
 

src/electron/userFilesInit.js

@@ -13,7 +13,6 @@ const { ensureDir } = require("./ensureDir");
  */
 exports.initUserFilesDir = (userDataDir) => {
   initDir("NOTES_DIR", path.join(userDataDir, "/notes"));
-  initDir("CACHE_DIR", userDataDir);
   initDir("SETTINGS_DIR", userDataDir);
 };
 

src/markdown/index.test.ts

@@ -2,6 +2,7 @@ import { expect } from "chai";
 import fs from "fs";
 import { describe, it } from "mocha";
 import path from "path";
+import yaml from "yaml";
 
 import { slateToString, stringToSlate } from "./index.js";
 import { dig, parseMarkdown, parseMarkdownForImport } from "./test-utils.js";
@@ -616,3 +617,43 @@ describe("Whacky shit", function () {
 
 ****[5 variations of Binary search (A Self Note)](https://leetcode.com/discuss/interview-question/1322500/5-variations-of-Binary-search-(A-Self-Note))****`;
 });
+
+describe("front matter parsing", function () {
+  const content = `---
+title: 2024-09-29
+tags: weekly-todo
+createdAt: 2024-09-30T17:50:22.000Z
+updatedAt: 2024-11-04T16:24:11.000Z
+---
+
+#weekly-todo
+
+Last week: [2024-09-22](../work/0193acd4fa3574698c36c4514b907c70.md)
+
+**I am on call this week** [On call week of 2024-09-30](../persona/0193acd4fa45731f81350d4443c1ed16.md)
+
+## Monday
+  
+`;
+
+  // A very basic "it works" test
+  // todo: End to end test with a real document, asserting against the database values
+  it("parses front matter as an mdast node, and can be parsed with yaml.parse", function () {
+    const parsed = parseMarkdown(content);
+    expect(parsed.children[0].type).to.equal("yaml");
+    expect(parsed.children[0].value).to.equal(
+      "title: 2024-09-29\n" +
+        "tags: weekly-todo\n" +
+        "createdAt: 2024-09-30T17:50:22.000Z\n" +
+        "updatedAt: 2024-11-04T16:24:11.000Z",
+    );
+
+    const frontMatter = yaml.parse(parsed.children[0].value as string);
+    expect(frontMatter).to.deep.equal({
+      title: "2024-09-29",
+      tags: "weekly-todo",
+      createdAt: "2024-09-30T17:50:22.000Z",
+      updatedAt: "2024-11-04T16:24:11.000Z",
+    });
+  });
+});

src/markdown/index.ts

@@ -5,8 +5,13 @@ import * as mdast from "mdast";
 export { slateToMdast } from "./remark-slate-transformer/transformers/slate-to-mdast.js";
 
 import { fromMarkdown } from "mdast-util-from-markdown";
+import {
+  frontmatterFromMarkdown,
+  frontmatterToMarkdown,
+} from "mdast-util-frontmatter";
 import { gfmFromMarkdown, gfmToMarkdown } from "mdast-util-gfm";
 import { toMarkdown } from "mdast-util-to-markdown";
+import { frontmatter } from "micromark-extension-frontmatter";
 import { gfm } from "micromark-extension-gfm";
 import { ofmTagFromMarkdown } from "./mdast-util-ofm-tag";
 import { ofmWikilinkFromMarkdown } from "./mdast-util-ofm-wikilink";
@@ -53,25 +58,28 @@ function wrapImages(tree: mdast.Root) {
 // to Chronicles tags and markdown links. Future versions may support these properly.
 export const parseMarkdownForImport = (markdown: string): mdast.Root => {
   return fromMarkdown(markdown, {
-    extensions: [gfm(), ofmTag(), ofmWikilink()],
+    extensions: [gfm(), ofmTag(), ofmWikilink(), frontmatter(["yaml"])],
     mdastExtensions: [
       gfmFromMarkdown(),
       ofmTagFromMarkdown(),
       ofmWikilinkFromMarkdown(),
+      // https://github.com/micromark/micromark-extension-frontmatter?tab=readme-ov-file#preset
+      // todo: support toml (need toml parser)
+      frontmatterFromMarkdown(["yaml"]),
     ],
   });
 };
 
 export const parseMarkdown = (markdown: string): mdast.Root => {
   return fromMarkdown(markdown, {
-    extensions: [gfm()],
-    mdastExtensions: [gfmFromMarkdown()],
+    extensions: [gfm(), frontmatter(["yaml"])],
+    mdastExtensions: [gfmFromMarkdown(), frontmatterFromMarkdown(["yaml"])],
   });
 };
 
 export const mdastToString = (tree: mdast.Nodes) => {
   return toMarkdown(tree, {
-    extensions: [gfmToMarkdown() as any],
+    extensions: [gfmToMarkdown() as any, frontmatterToMarkdown(["yaml"])],
     bullet: "-",
     emphasis: "_",
   });