update toktrie dep

parser/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.3.0"
 edition = "2021"
 
 [dependencies]
-toktrie = { git = "https://github.com/microsoft/toktrie", rev = "6172936f8c965d2050a53d14de0e3410ecc78ad1" }
+toktrie = { git = "https://github.com/microsoft/toktrie", rev = "148399b250ec9bbf0fc97149534035d8fad71164" }
 derivre = { git = "https://github.com/microsoft/derivre", rev = "02ee497e6e404a0b402b4f68a9abf599d22ed2ed" }
 serde = { version = "1.0.210", features = ["derive"] }
 serde_json = { version = "1.0.132", features = ["preserve_order"] }

sample_parser/Cargo.toml

@@ -6,7 +6,7 @@ default-run = "sample_parser"
 
 [dependencies]
 llguidance_parser = { path = "../parser" }
-toktrie_hf_tokenizers = { git = "https://github.com/microsoft/toktrie", rev = "6172936f8c965d2050a53d14de0e3410ecc78ad1" }
+toktrie_hf_tokenizers = { git = "https://github.com/microsoft/toktrie", rev = "148399b250ec9bbf0fc97149534035d8fad71164" }
 serde_json = "1.0.128"
 anyhow = "1.0.87"
 lazy_static = "1.5.0"

scripts/update-git.py

@@ -3,6 +3,9 @@
 import subprocess
 import re
 import sys
+import json
+import os
+
 
 # Function to get the latest commit hash of a git repository
 def get_latest_commit_hash(repo_path):
@@ -66,13 +69,41 @@ def update_cargo_toml(cargo_toml_path, toktrie_commit, derivre_commit):
 cargo_toml_paths = [
     "parser/Cargo.toml",
     "sample_parser/Cargo.toml",
+    "rust/Cargo.toml",
 ]
 
 # Update each Cargo.toml file
 for cargo_toml_path in cargo_toml_paths:
     update_cargo_toml(cargo_toml_path, toktrie_commit, derivre_commit)
 
-# Run cargo fetch for each path to update Cargo.lock
-subprocess.run(["cargo", "fetch", "--manifest-path", "rust/Cargo.toml"], check=True)
 
-print("All Cargo.toml files updated and cargo fetch run successfully.")
+def get_workspace_cargo_toml():
+    try:
+        result = subprocess.run(
+            ["cargo", "locate-project", "--workspace"],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        data = json.loads(result.stdout)
+        return data["root"]
+    except subprocess.CalledProcessError as e:
+        print("Error running cargo command:", e)
+    except KeyError:
+        print("Unexpected JSON structure from cargo command.")
+    return None
+
+
+ws = get_workspace_cargo_toml()
+if ws:
+    os.rename(ws, ws + ".tmp")
+
+try:
+    # Run cargo fetch for each path to update Cargo.lock
+    for path in cargo_toml_paths:
+        subprocess.run(["cargo", "fetch", "--manifest-path", path], check=True)
+finally:
+    if ws:
+        os.rename(ws + ".tmp", ws)
+
+print("All Cargo.toml files updated and cargo fetch run successfully.")