diff --git a/parser/src/earley/parser.rs b/parser/src/earley/parser.rs index 0bec6b9..60f7b1c 100644 --- a/parser/src/earley/parser.rs +++ b/parser/src/earley/parser.rs @@ -1552,6 +1552,10 @@ impl ParserState { return false; } if let Some(b) = transition_byte { + // At this point there may be a single-byte lexeme after the one + // we just recognized. For example, assuming C language, in the + // token "foo(", once we recognize the "foo" lexeme, we immediately + // have a single byte "(" lexeme. We deal with these here. let single = shared .lexer .check_for_single_byte_lexeme(no_hidden.lexer_state, b);