Token set is no longer off by one

microsoft/toktrie@d8d179a
guidance-ai · Jul 30, 2024 · 4718c2e · 4718c2e
1 parent f79d955
commit 4718c2e
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/rust/src/py.rs b/rust/src/py.rs
@@ -105,7 +105,6 @@ impl LLInterpreter {
                     .expect("expecting unconditional splice or mask");
                 let mut res = vec![0u8; m.len()];
                 m.iter_set_entries(|i| res[i] = 200);
-                res.pop();
                 Some(Cow::Owned(res))
             };