Comment on single byte lexemes (#52)

Comment on single byte lexemes.
microsoft · Nov 12, 2024 · 4de768e · 4de768e
1 parent 7f9ecb1
commit 4de768e
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/parser/src/earley/parser.rs b/parser/src/earley/parser.rs
@@ -1552,6 +1552,10 @@ impl ParserState {
                     return false;
                 }
                 if let Some(b) = transition_byte {
+                    // At this point there may be a single-byte lexeme after the one
+                    // we just recognized.  For example, assuming C language, in the
+                    // token "foo(", once we recognize the "foo" lexeme, we immediately
+                    // have a single byte "(" lexeme.  We deal with these here.
                     let single = shared
                         .lexer
                         .check_for_single_byte_lexeme(no_hidden.lexer_state, b);