From 7cfff79aa8e66a232c0aafa8f8586e542e1f0631 Mon Sep 17 00:00:00 2001 From: Xiaoxing Hu Date: Wed, 15 Nov 2023 14:44:07 +1300 Subject: [PATCH] stars, todo keyword and priority end after the whitespaces (#221) * [orga] stars, todo keyword and priority end after the whitespaces * add changeset --- .changeset/warm-seals-peel.md | 6 +++++ packages/editor/lib/plugins/cleanup.js | 4 ++-- .../footnote/stopped by headline.json | 4 ++-- .../src/__tests__/headline/broken drawer.json | 4 ++-- .../orga/src/__tests__/headline/drawers.json | 8 +++---- .../orga/src/__tests__/headline/keyword.json | 24 +++++++++---------- .../orga/src/__tests__/headline/nested.json | 24 +++++++++---------- .../orga/src/__tests__/headline/planning.json | 8 +++---- .../src/__tests__/headline/with tags.json | 12 +++++----- .../src/__tests__/keyword/multiple todo.json | 8 +++---- packages/orga/src/__tests__/keyword/todo.json | 8 +++---- packages/orga/src/parse/index.ts | 2 ++ packages/orga/src/tokenize/headline.test.ts | 22 ++++++++--------- packages/orga/src/tokenize/headline.ts | 19 +++++++++------ packages/orga/src/tokenize/index.ts | 2 ++ packages/orga/src/tokenize/inline.test.ts | 2 +- 16 files changed, 86 insertions(+), 71 deletions(-) create mode 100644 .changeset/warm-seals-peel.md diff --git a/.changeset/warm-seals-peel.md b/.changeset/warm-seals-peel.md new file mode 100644 index 00000000..f02a1dd6 --- /dev/null +++ b/.changeset/warm-seals-peel.md @@ -0,0 +1,6 @@ +--- +'orga': minor +'@orgajs/editor': patch +--- + +headline elements (stars, todo keywords and priority) end after the whitespaces diff --git a/packages/editor/lib/plugins/cleanup.js b/packages/editor/lib/plugins/cleanup.js index 79288971..8a39ff24 100644 --- a/packages/editor/lib/plugins/cleanup.js +++ b/packages/editor/lib/plugins/cleanup.js @@ -66,7 +66,7 @@ export const cleanupPlugin = ViewPlugin.define( }) }, }, - } + }, ) /** @@ -134,7 +134,7 @@ function createDecorations(view, decorations = Decoration.none) { add: [ Decoration.replace({ revealRange, - }).range(node.from, node.to + 1), // hide the space too + }).range(node.from, node.to), ], }) } diff --git a/packages/orga/src/__tests__/footnote/stopped by headline.json b/packages/orga/src/__tests__/footnote/stopped by headline.json index fd819ce7..44e8fb1e 100644 --- a/packages/orga/src/__tests__/footnote/stopped by headline.json +++ b/packages/orga/src/__tests__/footnote/stopped by headline.json @@ -136,8 +136,8 @@ }, "end": { "line": 3, - "column": 2, - "offset": 59 + "column": 3, + "offset": 60 } } }, diff --git a/packages/orga/src/__tests__/headline/broken drawer.json b/packages/orga/src/__tests__/headline/broken drawer.json index c4a79b46..0b6c9544 100644 --- a/packages/orga/src/__tests__/headline/broken drawer.json +++ b/packages/orga/src/__tests__/headline/broken drawer.json @@ -22,8 +22,8 @@ }, "end": { "line": 1, - "column": 2, - "offset": 1 + "column": 3, + "offset": 2 } } }, diff --git a/packages/orga/src/__tests__/headline/drawers.json b/packages/orga/src/__tests__/headline/drawers.json index 8d23cf98..1c329171 100644 --- a/packages/orga/src/__tests__/headline/drawers.json +++ b/packages/orga/src/__tests__/headline/drawers.json @@ -25,8 +25,8 @@ }, "end": { "line": 1, - "column": 2, - "offset": 1 + "column": 3, + "offset": 2 } } }, @@ -237,8 +237,8 @@ }, "end": { "line": 7, - "column": 2, - "offset": 77 + "column": 3, + "offset": 78 } } }, diff --git a/packages/orga/src/__tests__/headline/keyword.json b/packages/orga/src/__tests__/headline/keyword.json index 1a8089a2..c4fa7239 100644 --- a/packages/orga/src/__tests__/headline/keyword.json +++ b/packages/orga/src/__tests__/headline/keyword.json @@ -54,8 +54,8 @@ }, "end": { "line": 3, - "column": 2, - "offset": 27 + "column": 3, + "offset": 28 } } }, @@ -71,8 +71,8 @@ }, "end": { "line": 3, - "column": 7, - "offset": 32 + "column": 8, + "offset": 33 } } }, @@ -142,8 +142,8 @@ }, "end": { "line": 4, - "column": 2, - "offset": 47 + "column": 3, + "offset": 48 } } }, @@ -159,8 +159,8 @@ }, "end": { "line": 4, - "column": 7, - "offset": 52 + "column": 8, + "offset": 53 } } }, @@ -230,8 +230,8 @@ }, "end": { "line": 5, - "column": 2, - "offset": 67 + "column": 3, + "offset": 68 } } }, @@ -247,8 +247,8 @@ }, "end": { "line": 5, - "column": 7, - "offset": 72 + "column": 8, + "offset": 73 } } }, diff --git a/packages/orga/src/__tests__/headline/nested.json b/packages/orga/src/__tests__/headline/nested.json index 65703e5b..0db57d68 100644 --- a/packages/orga/src/__tests__/headline/nested.json +++ b/packages/orga/src/__tests__/headline/nested.json @@ -22,8 +22,8 @@ }, "end": { "line": 1, - "column": 2, - "offset": 1 + "column": 3, + "offset": 2 } } }, @@ -127,8 +127,8 @@ }, "end": { "line": 3, - "column": 3, - "offset": 25 + "column": 4, + "offset": 26 } } }, @@ -183,8 +183,8 @@ }, "end": { "line": 4, - "column": 4, - "offset": 42 + "column": 5, + "offset": 43 } } }, @@ -331,8 +331,8 @@ }, "end": { "line": 7, - "column": 3, - "offset": 69 + "column": 4, + "offset": 70 } } }, @@ -415,8 +415,8 @@ }, "end": { "line": 8, - "column": 2, - "offset": 84 + "column": 3, + "offset": 85 } } }, @@ -471,8 +471,8 @@ }, "end": { "line": 9, - "column": 3, - "offset": 98 + "column": 4, + "offset": 99 } } }, diff --git a/packages/orga/src/__tests__/headline/planning.json b/packages/orga/src/__tests__/headline/planning.json index d9699938..984ed8cb 100644 --- a/packages/orga/src/__tests__/headline/planning.json +++ b/packages/orga/src/__tests__/headline/planning.json @@ -22,8 +22,8 @@ }, "end": { "line": 1, - "column": 2, - "offset": 1 + "column": 3, + "offset": 2 } } }, @@ -162,8 +162,8 @@ }, "end": { "line": 4, - "column": 2, - "offset": 45 + "column": 3, + "offset": 46 } } }, diff --git a/packages/orga/src/__tests__/headline/with tags.json b/packages/orga/src/__tests__/headline/with tags.json index 161b0970..68aed28f 100644 --- a/packages/orga/src/__tests__/headline/with tags.json +++ b/packages/orga/src/__tests__/headline/with tags.json @@ -22,8 +22,8 @@ }, "end": { "line": 1, - "column": 2, - "offset": 1 + "column": 3, + "offset": 2 } } }, @@ -78,8 +78,8 @@ }, "end": { "line": 2, - "column": 3, - "offset": 14 + "column": 4, + "offset": 15 } } }, @@ -157,8 +157,8 @@ }, "end": { "line": 3, - "column": 4, - "offset": 40 + "column": 5, + "offset": 41 } } }, diff --git a/packages/orga/src/__tests__/keyword/multiple todo.json b/packages/orga/src/__tests__/keyword/multiple todo.json index 03908f5b..52489fee 100644 --- a/packages/orga/src/__tests__/keyword/multiple todo.json +++ b/packages/orga/src/__tests__/keyword/multiple todo.json @@ -88,8 +88,8 @@ }, "end": { "line": 5, - "column": 2, - "offset": 87 + "column": 3, + "offset": 88 } } }, @@ -105,8 +105,8 @@ }, "end": { "line": 5, - "column": 8, - "offset": 93 + "column": 9, + "offset": 94 } } }, diff --git a/packages/orga/src/__tests__/keyword/todo.json b/packages/orga/src/__tests__/keyword/todo.json index 6fd71932..6c8e774c 100644 --- a/packages/orga/src/__tests__/keyword/todo.json +++ b/packages/orga/src/__tests__/keyword/todo.json @@ -54,8 +54,8 @@ }, "end": { "line": 3, - "column": 2, - "offset": 27 + "column": 3, + "offset": 28 } } }, @@ -71,8 +71,8 @@ }, "end": { "line": 3, - "column": 7, - "offset": 32 + "column": 8, + "offset": 33 } } }, diff --git a/packages/orga/src/parse/index.ts b/packages/orga/src/parse/index.ts index a1ee987e..b49a6df9 100644 --- a/packages/orga/src/parse/index.ts +++ b/packages/orga/src/parse/index.ts @@ -186,6 +186,8 @@ export function parser(lexer: Lexer, options: ParserOptions): Parser { function finish() { context.exitTo('document') context.exit('document') + // algin the end position + context.tree.position.end = lexer.toPoint(lexer.now) return context.tree } diff --git a/packages/orga/src/tokenize/headline.test.ts b/packages/orga/src/tokenize/headline.test.ts index b372beb3..ca57798d 100644 --- a/packages/orga/src/tokenize/headline.test.ts +++ b/packages/orga/src/tokenize/headline.test.ts @@ -5,12 +5,12 @@ import tokenize from './__tests__/tok' describe('tokenize headline', () => { it('knows headlines', () => { assert.deepEqual(tokenize('** a headline'), [ - { _text: '**', level: 2, type: 'stars' }, + { _text: '** ', level: 2, type: 'stars' }, { _text: 'a headline', type: 'text', value: 'a headline' }, ]) assert.deepEqual(tokenize('** _headline_'), [ - { _text: '**', level: 2, type: 'stars' }, + { _text: '** ', level: 2, type: 'stars' }, { _text: '_headline_', style: 'underline', @@ -20,29 +20,29 @@ describe('tokenize headline', () => { ]) assert.deepEqual(tokenize('** a headline'), [ - { _text: '**', level: 2, type: 'stars' }, + { _text: '** ', level: 2, type: 'stars' }, { _text: 'a headline', type: 'text', value: 'a headline' }, ]) assert.deepEqual(tokenize('***** a headline'), [ - { _text: '*****', level: 5, type: 'stars' }, + { _text: '***** ', level: 5, type: 'stars' }, { _text: 'a headline', type: 'text', value: 'a headline' }, ]) assert.deepEqual(tokenize('* a 😀line'), [ - { _text: '*', level: 1, type: 'stars' }, + { _text: '* ', level: 1, type: 'stars' }, { _text: 'a 😀line', type: 'text', value: 'a 😀line' }, ]) assert.deepEqual(tokenize('* TODO [#A] a headline :tag1:tag2:'), [ - { _text: '*', level: 1, type: 'stars' }, + { _text: '* ', level: 1, type: 'stars' }, { - _text: 'TODO', + _text: 'TODO ', actionable: true, keyword: 'TODO', type: 'todo', }, - { _text: '[#A]', type: 'priority', value: '[#A]' }, + { _text: '[#A] ', type: 'priority', value: '[#A]' }, { _text: 'a headline', type: 'text', value: 'a headline' }, { _text: ':tag1:tag2:', @@ -56,14 +56,14 @@ describe('tokenize headline', () => { '* TODO [#A] a headline :tag1:123:#hash:@at:org-mode:under_score:98%:' ), [ - { _text: '*', level: 1, type: 'stars' }, + { _text: '* ', level: 1, type: 'stars' }, { - _text: 'TODO', + _text: 'TODO ', actionable: true, keyword: 'TODO', type: 'todo', }, - { _text: '[#A]', type: 'priority', value: '[#A]' }, + { _text: '[#A] ', type: 'priority', value: '[#A]' }, { _text: 'a headline', type: 'text', value: 'a headline' }, { _text: ':tag1:123:#hash:@at:org-mode:under_score:98%:', diff --git a/packages/orga/src/tokenize/headline.ts b/packages/orga/src/tokenize/headline.ts index f42333e0..3c677429 100644 --- a/packages/orga/src/tokenize/headline.ts +++ b/packages/orga/src/tokenize/headline.ts @@ -28,30 +28,35 @@ export default (getTodoKeywordSets: GetTodoKeywordSets) => buffer.push({ type: 'stars', level: stars.value.length, - position: stars.position, + position: { + start: stars.position.start, + end: eat('whitespaces').position.end, + }, }) - - eat('whitespaces') const keyword = eat(RegExp(`${todos.map(escape).join('|')}(?=\\s)`, 'y')) if (keyword) { buffer.push({ type: 'todo', keyword: keyword.value, actionable: isActionable(keyword.value), - position: keyword.position, + position: { + start: keyword.position.start, + end: eat('whitespaces').position.end, + }, }) } - eat('whitespaces') const priority = eat(/^\[#(A|B|C)\](?=\s)/y) if (priority) { buffer.push({ type: 'priority', ...priority, + position: { + start: priority.position.start, + end: eat('whitespaces').position.end, + }, }) } - eat('whitespaces') - const tags = match(/[ \t]+(:(?:[\w@_#%-]+:)+)[ \t]*$/m, { end: endOfLine(), }) diff --git a/packages/orga/src/tokenize/index.ts b/packages/orga/src/tokenize/index.ts index 986583db..602b6369 100644 --- a/packages/orga/src/tokenize/index.ts +++ b/packages/orga/src/tokenize/index.ts @@ -33,6 +33,7 @@ export interface Lexer { modify(f: (t: Token) => Token, offset?: number): void readonly now: number toOffset: (point: Point | number) => number + toPoint: (point: number) => Point } export type Tokenizer = (reader: Reader) => Token[] | Token | void @@ -167,5 +168,6 @@ export const tokenize = (text: string, options: LexerOptions): Lexer => { return reader.toIndex(token?.position.start ?? Infinity) }, toOffset: (point) => reader.toIndex(point), + toPoint: (offset) => reader.toPoint(offset), } } diff --git a/packages/orga/src/tokenize/inline.test.ts b/packages/orga/src/tokenize/inline.test.ts index cdb400b9..41db7c88 100644 --- a/packages/orga/src/tokenize/inline.test.ts +++ b/packages/orga/src/tokenize/inline.test.ts @@ -368,7 +368,7 @@ describe('Inline Tokenization', () => { it('recon invalid inline markups', () => { assert.deepEqual(tokenize(`* word*`), [ { - _text: '*', + _text: '* ', level: 1, type: 'stars', },