Port guidance's JSON implementation (#48)

Port the JSON grammar builder from guidance --------- Co-authored-by: Michal Moskal <[email protected]>
microsoft · Nov 20, 2024 · fdd515e · fdd515e
1 parent 7a98040
commit fdd515e
Show file tree

Hide file tree

Showing 21 changed files with 2,493 additions and 646 deletions.
diff --git a/parser/Cargo.lock b/parser/Cargo.lock
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
@@ -15,6 +15,8 @@ jsonschema = { version = "0.24.0", default-features = false, optional = true }
 url = "2.5.2"
 lazy_static = { version = "1.5.0", optional = true }
 regex-syntax = "0.8.5"
+indexmap = "2.6.0"
+referencing = "0.26.1"
 rayon = { version = "1.10.0", optional = true }
 
 [features]

diff --git a/parser/build.rs b/parser/build.rs
@@ -25,7 +25,10 @@ fn main() {
             },
             |bindings| {
                 bindings.write_to_file("llguidance.h");
-                bindings.write_to_file(format!("{}/../../../llguidance.h", env::var("OUT_DIR").unwrap()));
+                bindings.write_to_file(format!(
+                    "{}/../../../llguidance.h",
+                    env::var("OUT_DIR").unwrap()
+                ));
             },
         );
 }
diff --git a/parser/src/earley/from_guidance.rs b/parser/src/earley/from_guidance.rs
@@ -93,12 +93,12 @@ fn grammar_from_json(
             "cannot have both json_schema/lark_grammar and nodes/rx_nodes"
         );
 
-        let mut new_grm = if let Some(json_schema) = input.json_schema.as_ref() {
+        let mut new_grm = if let Some(json_schema) = input.json_schema.take() {
             ensure!(
                 input.lark_grammar.is_none(),
                 "cannot have both json_schema and lark_grammar"
             );
-            let opts = JsonCompileOptions { compact: false };
+            let opts: JsonCompileOptions = JsonCompileOptions::default();
             opts.json_to_llg(json_schema)?
         } else {
             lark_to_llguidance(input.lark_grammar.as_ref().unwrap())?

diff --git a/parser/src/ffi.rs b/parser/src/ffi.rs
@@ -356,9 +356,9 @@ fn new_constraint_json(init: &LlgConstraintInit, json_schema: *const c_char) ->
         .map_err(|_| anyhow::anyhow!("Invalid UTF-8 in json_schema"))?;
     let json_schema = serde_json::from_str(json_schema)
         .map_err(|e| anyhow::anyhow!("Invalid JSON in json_schema: {e}"))?;
-    let opts = JsonCompileOptions { compact: false };
+    let opts = JsonCompileOptions::default();
     let grammar = opts
-        .json_to_llg(&json_schema)
+        .json_to_llg(json_schema)
         .map_err(|e| anyhow::anyhow!("Error compiling JSON schema to LLG: {e}"))?;
     init.build_constraint(grammar)
 }

diff --git a/parser/src/grammar_builder.rs b/parser/src/grammar_builder.rs
@@ -7,7 +7,7 @@ use crate::api::{
     RegexSpec, TopLevelGrammar,
 };
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
 pub struct NodeRef {
     idx: usize,
     grammar_id: u32,
@@ -91,6 +91,14 @@ impl RegexBuilder {
         self.add_node(RegexNode::Repeat(node, min, max))
     }
 
+    pub fn not(&mut self, node: RegexId) -> RegexId {
+        self.add_node(RegexNode::Not(node))
+    }
+
+    pub fn and(&mut self, nodes: Vec<RegexId>) -> RegexId {
+        self.add_node(RegexNode::And(nodes))
+    }
+
     fn finalize(&mut self) -> Vec<RegexNode> {
         let r = std::mem::take(&mut self.nodes);
         *self = Self::new();