Comment on call stack for committing tokens (#50)

* Comments on call stack for committing tokens * Comment on call stack for committing tokens --------- Co-authored-by: Jeffrey Kegler <[email protected]>
microsoft · Nov 12, 2024 · a81808e · a81808e
1 parent 5114648
commit a81808e
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 3 deletions.
diff --git a/parser/src/earley/parser.rs b/parser/src/earley/parser.rs
@@ -1716,6 +1716,9 @@ impl Parser {
         Ok(Parser { shared, state })
     }
 
+    /// This is a top-level method in this file.  It is called by mid_process_inner()
+    /// in TokenParser in tokenparser.rs.  It is used by the mid_process() method of
+    /// the LLInterpreter interface.
     pub fn compute_bias_after_gen_grammar(
         &mut self,
         computer: &dyn BiasComputer,
@@ -1727,6 +1730,9 @@ impl Parser {
         (self.state.trie_gen_grammar_accepting, r)
     }
 
+    /// This is a top-level method in this file.  It is called by mid_process_inner()
+    /// in TokenParser in tokenparser.rs.  It is used by the mid_process() method of
+    /// the LLInterpreter interface.
     pub fn compute_bias(&mut self, computer: &dyn BiasComputer, start: &[u8]) -> SimpleVob {
         let mut shared = self.shared.lock().unwrap();
         self.state.compute_bias(&mut shared, computer, start)

diff --git a/parser/src/tokenparser.rs b/parser/src/tokenparser.rs
@@ -258,9 +258,10 @@ impl TokenParser {
         self.error_message.clone()
     }
 
-    // advance_parser() is a top-level method in this file.  
-    // This advance_parser() is called indirectly via the advance_parser() method
-    // of the llguidance LLInterpreter interface,
+    // advance_parser() is a top-level method in this file.
+    // This advance_parser() is called by Constraint::commit_token().
+    // It is accessible via the advance_parser() method of
+    // the LLInterpreter interface.
     //
     // The result here *never* includes a mask.
     // It's either stop or an unconditional splice (possibly noop).
@@ -277,6 +278,11 @@ impl TokenParser {
         r
     }
 
+    // mid_process() is a top-level method in this file.
+    // mid_process() is called by Constraint::commit_token().
+    // It is also be called by TokenParser::advance_parser()
+    // within this file, in which case it is accessible
+    // via the advance_parser() method of the LLInterpreter interface.
     pub fn mid_process(&mut self, mut arg: StepArg) -> StepResult {
         assert!(self.is_fresh == false, "process_prompt() not called");