Tokenize combining marks as WordChars not Symbol.

Closes #114.
jgm · Oct 20, 2023 · cf945af · cf945af
1 parent 68374fd
commit cf945af
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 3 deletions.
diff --git a/commonmark/src/Commonmark/Tokens.hs b/commonmark/src/Commonmark/Tokens.hs
@@ -10,7 +10,7 @@ module Commonmark.Tokens
   , untokenize
   ) where
 
-import           Unicode.Char    (isAlphaNum)
+import           Unicode.Char    (isAlphaNum, isMark)
 import           Unicode.Char.General.Compat  (isSpace)
 import           Data.Text       (Text)
 import qualified Data.Text       as T
@@ -42,7 +42,9 @@ tokenize name =
     -- everything else gets in a token by itself.
     f '\r' '\n' = True
     f ' ' ' '   = True
-    f x   y     = isAlphaNum x && isAlphaNum y
+    f x   y     = isWordChar x && isWordChar y
+
+    isWordChar c  = isAlphaNum c || isMark c
 
     go !_pos [] = []
     go !pos (!t:ts) = -- note that t:ts are guaranteed to be nonempty
@@ -57,7 +59,7 @@ tokenize name =
          '\n' -> Tok LineEnd pos t :
                  go (incSourceLine (setSourceColumn pos 1) 1) ts
          thead
-           | isAlphaNum thead ->
+           | isWordChar thead ->
                  Tok WordChars pos t :
                  go (incSourceColumn pos (T.length t)) ts
            | isSpace thead ->

diff --git a/commonmark/test/regression.md b/commonmark/test/regression.md
@@ -203,3 +203,10 @@ Issue #67.
 <p><a href="http://www.example.com/">test </a>
 <a href="http://www.example.com/"> test</a></p>
 ````````````````````````````````
+
+Issue #114.
+```````````````````````````````` example
+*.*̀.
+.
+<p>*.*̀.</p>
+````````````````````````````````