diff --git a/parser/src/earley/parser.rs b/parser/src/earley/parser.rs index 33e37a1..0bec6b9 100644 --- a/parser/src/earley/parser.rs +++ b/parser/src/earley/parser.rs @@ -1496,11 +1496,15 @@ impl ParserState { } } - /// Advance the parser with given lexeme_idx. - /// lexer_state is state *after* consuming the byte. - /// It either initial lexer states for lazy lexers, - /// or lexer_initial_state+byte for greedy lexers. - /// lexer_byte is the byte that led to producing the lexeme. + + /// Advance the parser with given 'pre_lexeme'. + /// On return, the lexer_state will be the state *after* consuming + /// 'pre_lexeme'. As a special case, a following single byte lexeme + /// is also consumed. + /// + // The new lexer state will be an initial lexer states when the lexing + // is lazy. If the lexing was greedy, it will be an initial lexer state + // advanced to the byte which produced the greedy lexeme. // This is never inlined anyways, so better make it formal #[inline(never)]