drop GIL for mid_process()

guidance-ai · Jul 31, 2024 · 5e66bc7 · 5e66bc7
1 parent 20e7fae
commit 5e66bc7
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/rust/src/py.rs b/rust/src/py.rs
@@ -81,9 +81,9 @@ impl LLInterpreter {
         self.inner.process_prompt(prompt)
     }
 
-    fn mid_process(&mut self) -> (Option<Cow<[u8]>>, String) {
+    fn mid_process(&mut self, py: Python<'_>) -> (Option<Cow<[u8]>>, String) {
         let arg = std::mem::replace(&mut self.step_arg, StepArg::empty());
-        self.last_result = self.inner.mid_process(arg);
+        self.last_result = py.allow_threads(|| self.inner.mid_process(arg));
         let r = &self.last_result;
         let is_final = r.is_stop();
         if let Some(t) = r.temperature {