From 09d8d590504b5ef8eb9aa03324805327ec89cd2b Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 25 Oct 2024 11:43:35 -0700
Subject: [PATCH 01/73] implement schema JSON -> llgrammar JSON

---
 python/llguidance/__init__.py |  3 ++-
 python/llguidance/_lib.pyi    | 21 ++++++++++++++++++++-
 rust/src/py.rs                | 30 +++++++++++++++++++++++++++++-
 3 files changed, 51 insertions(+), 3 deletions(-)

diff --git a/python/llguidance/__init__.py b/python/llguidance/__init__.py
index d171c8d..0726170 100644
--- a/python/llguidance/__init__.py
+++ b/python/llguidance/__init__.py
@@ -1,8 +1,9 @@
-from ._lib import LLTokenizer, LLInterpreter
+from ._lib import LLTokenizer, LLInterpreter, JsonCompiler
 from ._tokenizer import TokenizerWrapper
 
 __all__ = [
     "LLTokenizer",
     "LLInterpreter",
+    "JsonCompiler",
     "TokenizerWrapper",
 ]
diff --git a/python/llguidance/_lib.pyi b/python/llguidance/_lib.pyi
index c05bc61..ee7f776 100644
--- a/python/llguidance/_lib.pyi
+++ b/python/llguidance/_lib.pyi
@@ -138,4 +138,23 @@ class LLInterpreter:
     def has_pending_stop(self) -> bool:
         """
         If true, next mid_process() call will return stop
-        """
\ No newline at end of file
+        """
+
+class JsonCompiler:
+    def __new__(
+        cls,
+        compact: bool = False,
+    ) -> "JsonCompiler":
+        """
+        Create a new JSON compiler.
+        Args:
+            compact: bool - whether to use compact JSON representation
+        """
+
+    def compile(
+        self,
+        schema: str,
+    ) -> str:
+        """
+        Compile the JSON representation of the AG2 grammar/constraint.
+        """
diff --git a/rust/src/py.rs b/rust/src/py.rs
index 2c423e9..d49e06a 100644
--- a/rust/src/py.rs
+++ b/rust/src/py.rs
@@ -6,9 +6,10 @@ use llguidance_parser::toktrie::{
     self, InferenceCapabilities, TokRxInfo, TokTrie, TokenId, TokenizerEnv,
 };
 use llguidance_parser::{api::TopLevelGrammar, output::ParserOutput, TokenParser};
-use llguidance_parser::{Constraint, Logger};
+use llguidance_parser::{Constraint, JsonCompileOptions, Logger};
 use pyo3::{exceptions::PyValueError, prelude::*};
 use serde::{Deserialize, Serialize};
+use serde_json::Value;
 
 #[derive(Clone)]
 #[pyclass]
@@ -240,9 +241,36 @@ impl TokenizerEnv for LLTokenizer {
     }
 }
 
+#[derive(Clone)]
+#[pyclass]
+struct JsonCompiler {
+    #[pyo3(get, set)]
+    compact: bool,
+}
+
+#[pymethods]
+impl JsonCompiler {
+    #[new]
+    fn py_new(compact: Option<bool>) -> Self {
+        JsonCompiler {
+            compact: compact.unwrap_or(false),
+        }
+    }
+    fn compile(&self, schema: &str) -> PyResult<String> {
+        let schema: Value = serde_json::from_str(schema).map_err(val_error)?;
+        let compile_options = JsonCompileOptions {
+            compact: self.compact,
+        };
+        let grammar = compile_options.json_to_llg(&schema).map_err(val_error)?;
+        Ok(serde_json::to_string(&grammar).map_err(val_error)?)
+    }
+
+}
+
 pub(crate) fn init(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<LLTokenizer>()?;
     m.add_class::<LLInterpreter>()?;
+    m.add_class::<JsonCompiler>()?;
     Ok(())
 }
 

From 78ccc30e7a9ff28d4478c76833379c660278c85b Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Thu, 7 Nov 2024 12:32:57 -0800
Subject: [PATCH 02/73] Return GrammarWithLexer instead of TopLevelGrammar

---
 rust/src/py.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/rust/src/py.rs b/rust/src/py.rs
index d49e06a..a0d38b6 100644
--- a/rust/src/py.rs
+++ b/rust/src/py.rs
@@ -261,8 +261,9 @@ impl JsonCompiler {
         let compile_options = JsonCompileOptions {
             compact: self.compact,
         };
-        let grammar = compile_options.json_to_llg(&schema).map_err(val_error)?;
-        Ok(serde_json::to_string(&grammar).map_err(val_error)?)
+        let tlg = compile_options.json_to_llg(&schema).map_err(val_error)?;
+        let grammar = &tlg.grammars[0];
+        Ok(serde_json::to_string(grammar).map_err(val_error)?)
     }
 
 }

From b30d5e85c992e6c929d41de450bff289018b3da3 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Thu, 7 Nov 2024 13:11:19 -0800
Subject: [PATCH 03/73] refactor json into mod

---
 parser/src/{json.rs => json/compiler.rs} | 0
 parser/src/json/mod.rs                   | 1 +
 parser/src/lib.rs                        | 2 +-
 3 files changed, 2 insertions(+), 1 deletion(-)
 rename parser/src/{json.rs => json/compiler.rs} (100%)
 create mode 100644 parser/src/json/mod.rs

diff --git a/parser/src/json.rs b/parser/src/json/compiler.rs
similarity index 100%
rename from parser/src/json.rs
rename to parser/src/json/compiler.rs
diff --git a/parser/src/json/mod.rs b/parser/src/json/mod.rs
new file mode 100644
index 0000000..59d8df7
--- /dev/null
+++ b/parser/src/json/mod.rs
@@ -0,0 +1 @@
+pub mod compiler;
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index c33c4ad..5b1d84f 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -27,7 +27,7 @@ pub mod lark;
 mod grammar_builder;
 mod json;
 pub use grammar_builder::{GrammarBuilder, NodeRef};
-pub use json::JsonCompileOptions;
+pub use json::compiler::JsonCompileOptions;
 pub use tokenizer_json::token_bytes_from_tokenizer_json;
 
 #[macro_export]

From 5f836aef4ebbd6da901eac6f9a0c6029f39b1534 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Thu, 7 Nov 2024 16:25:11 -0800
Subject: [PATCH 04/73] default to all types if type unspecified

---
 parser/src/json/compiler.rs | 55 +++++++++++++++++++------------------
 1 file changed, 29 insertions(+), 26 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index cd09798..9d266f5 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -22,6 +22,7 @@ fn to_compact_json(target: &serde_json::Value) -> String {
     serde_json::to_string(target).unwrap()
 }
 
+const TYPES: [&str; 7] = ["null", "boolean", "integer", "number", "string", "array", "object"];
 const KEYWORDS: [&str; 10] = [
     "anyOf",
     "oneOf",
@@ -60,6 +61,7 @@ const DEFS_KEYS: [&str; 4] = ["$defs", "definitions", "defs", "refs"];
 
 const ARRAY_KEYS: [&str; 4] = ["items", "prefixItems", "minItems", "maxItems"];
 const OBJECT_KEYS: [&str; 2] = ["properties", "additionalProperties"];
+const STRING_KEYS: [&str; 4] = ["minLength", "maxLength", "pattern", "format"];
 
 const CHAR_REGEX: &str = r#"(\\([\"\\\/bfnrt]|u[a-fA-F0-9]{4})|[^\"\\\x00-\x1F\x7F])"#;
 
@@ -78,17 +80,9 @@ fn validate_json_node_keys(node: &Value) -> Result<()> {
         .ok_or_else(|| anyhow!("Expected object as json schema, got: {}", limited_str(node)))
         .unwrap();
 
-    let typ = node.get("type").and_then(|v| v.as_str()).unwrap_or("");
-
     for key in node.keys() {
         let key = &key.as_str();
-        if KEYWORDS.contains(key) || IGNORED_KEYS.contains(key) || DEFS_KEYS.contains(key) {
-            continue;
-        }
-        if typ == "array" && ARRAY_KEYS.contains(key) {
-            continue;
-        }
-        if typ == "object" && OBJECT_KEYS.contains(key) {
+        if KEYWORDS.contains(key) || IGNORED_KEYS.contains(key) || DEFS_KEYS.contains(key) || ARRAY_KEYS.contains(key) || OBJECT_KEYS.contains(key) || STRING_KEYS.contains(key) {
             continue;
         }
         if key.starts_with("x-") || key.starts_with("$xsd-") {
@@ -299,6 +293,13 @@ impl Compiler {
             bail!("'false' not supported as schema here");
         }
 
+        if let Some(json_schema) = json_schema.as_object() {
+            // TODO: should be sufficient to have only ignored keys here
+            if json_schema.is_empty() {
+                return Ok(self.gen_json_any());
+            }
+        }
+
         // eprintln!("gen_json: {}", limited_str(json_schema));
         validate_json_node_keys(json_schema)?;
 
@@ -343,26 +344,28 @@ impl Compiler {
             return Ok(self.builder.select(&options));
         }
 
-        // Process type-specific keywords
-        if let Some(arr) = json_schema["type"].as_array() {
-            let nodes = arr
-                .iter()
-                .map(|v| {
-                    let tp = v.as_str().ok_or_else(|| {
-                        anyhow!("Expected string in type list, got: {}", limited_str(v))
-                    })?;
-                    self.gen_json_type(tp, json_schema)
-                })
-                .collect::<Result<Vec<_>>>()?;
-            return Ok(self.builder.select(&nodes));
+        // Process type
+        if let Some(tp) = json_schema.opt_str("type")? {
+            return self.gen_json_type(tp, json_schema)
         }
 
-        if let Some(target_type_str) = json_schema.opt_str("type")? {
-            return self.gen_json_type(target_type_str, json_schema);
-        }
+        let types = match json_schema.opt_array("type")? {
+            Some(types) => types,
+            None => {
+                &TYPES.iter().map(|s| Value::String(s.to_string())).collect::<Vec<_>>()
+            }
+        };
 
-        // Fallback to "any" type
-        Ok(self.gen_json_any())
+        let nodes = types
+            .iter()
+            .map(|v| {
+                let tp = v.as_str().ok_or_else(|| {
+                    anyhow!("Expected string in type list, got: {}", limited_str(v))
+                })?;
+                self.gen_json_type(tp, json_schema)
+            })
+            .collect::<Result<Vec<_>>>()?;
+        Ok(self.builder.select(&nodes))
     }
 
     fn gen_json_type(&mut self, target_type_str: &str, json_schema: &Value) -> Result<NodeRef> {

From e67ed0fc68aea50289d2a68a742434eb33b94005 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Thu, 7 Nov 2024 16:26:09 -0800
Subject: [PATCH 05/73] basic formats

---
 parser/src/json/compiler.rs | 19 ++++++++++++++++++-
 parser/src/json/formats.rs  | 17 +++++++++++++++++
 parser/src/json/mod.rs      |  1 +
 3 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 parser/src/json/formats.rs

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index 9d266f5..e7ab928 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -4,6 +4,7 @@ use lazy_static::lazy_static;
 use serde_json::{json, Value};
 use std::{collections::HashMap, vec};
 
+use super::formats::lookup_format;
 use crate::{
     api::{GrammarWithLexer, RegexSpec, TopLevelGrammar},
     GrammarBuilder, NodeRef,
@@ -378,7 +379,8 @@ impl Compiler {
                 let min_length = json_schema.opt_u64("minLength")?.unwrap_or(0);
                 let max_length = json_schema.opt_u64("maxLength")?;
                 let pattern = json_schema.opt_str("pattern")?;
-                return self.gen_json_string(min_length, max_length, pattern);
+                let format = json_schema.opt_str("format")?;
+                return self.gen_json_string(min_length, max_length, pattern, format);
             }
             "array" => {
                 let empty = vec![];
@@ -520,7 +522,22 @@ impl Compiler {
         min_length: u64,
         max_length: Option<u64>,
         regex: Option<&str>,
+        format: Option<&str>,
     ) -> Result<NodeRef> {
+
+        let mut regex = regex;
+
+        if let Some(format) = format {
+            if regex.is_some() {
+                bail!("Cannot specify both a regex and a format for a JSON string");
+            }
+            if let Some(r) = lookup_format(format) {
+                regex = Some(r);
+            } else {
+                bail!("Unknown format: {}", format)
+            };
+        }
+
         if min_length == 0 && max_length.is_none() && regex.is_none() {
             return Ok(self.json_simple_string());
         }
diff --git a/parser/src/json/formats.rs b/parser/src/json/formats.rs
new file mode 100644
index 0000000..bce6173
--- /dev/null
+++ b/parser/src/json/formats.rs
@@ -0,0 +1,17 @@
+use lazy_static::lazy_static;
+use std::collections::HashMap;
+
+lazy_static! {
+    static ref FORMAT_PATTERNS: HashMap<&'static str, &'static str> = {
+        let mut m = HashMap::new();
+        m.insert("email", r"^[^\s@]+@[^\s@]+\.[^\s@]+$");
+        m.insert("ipv4", r"^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$");
+        m.insert("date", r"^\d{4}-\d{2}-\d{2}$");
+        // Add more patterns as needed
+        m
+    };
+}
+
+pub fn lookup_format(name: &str) -> Option<&str> {
+    FORMAT_PATTERNS.get(name).copied()
+}
diff --git a/parser/src/json/mod.rs b/parser/src/json/mod.rs
index 59d8df7..3c89888 100644
--- a/parser/src/json/mod.rs
+++ b/parser/src/json/mod.rs
@@ -1 +1,2 @@
 pub mod compiler;
+mod formats;

From 261d27ebb4a635d9d1fc052f9cd9a975de6c312f Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Thu, 7 Nov 2024 16:52:34 -0800
Subject: [PATCH 06/73] format parity

---
 parser/src/json/formats.rs | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/parser/src/json/formats.rs b/parser/src/json/formats.rs
index bce6173..e8924d3 100644
--- a/parser/src/json/formats.rs
+++ b/parser/src/json/formats.rs
@@ -3,12 +3,18 @@ use std::collections::HashMap;
 
 lazy_static! {
     static ref FORMAT_PATTERNS: HashMap<&'static str, &'static str> = {
-        let mut m = HashMap::new();
-        m.insert("email", r"^[^\s@]+@[^\s@]+\.[^\s@]+$");
-        m.insert("ipv4", r"^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$");
-        m.insert("date", r"^\d{4}-\d{2}-\d{2}$");
-        // Add more patterns as needed
-        m
+        HashMap::from([
+            ("date-time", r"(?P<date>[0-9]{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01]))[tT](?P<time>(?:[01][0-9]|2[0-3]):[0-5][0-9]:(?:[0-5][0-9]|60)(?P<time_fraction>\.[0-9]+)?(?P<time_zone>[zZ]|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9]))"),
+            ("time", r"(?:[01][0-9]|2[0-3]):[0-5][0-9]:(?:[0-5][0-9]|60)(?P<time_fraction>\.[0-9]+)?(?P<time_zone>[zZ]|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9])"),
+            ("date", r"[0-9]{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])"),
+            ("duration", r"P(?:(?P<dur_date>(?:(?P<dur_year>[0-9]+Y(?:[0-9]+M(?:[0-9]+D)?)?)|(?P<dur_month>[0-9]+M(?:[0-9]+D)?)|(?P<dur_day>[0-9]+D))(?:T(?:(?P<dur_hour>[0-9]+H(?:[0-9]+M(?:[0-9]+S)?)?)|(?P<dur_minute>[0-9]+M(?:[0-9]+S)?)|(?P<dur_second>[0-9]+S)))?)|(?P<dur_time>T(?:(?P<dur_hour2>[0-9]+H(?:[0-9]+M(?:[0-9]+S)?)?)|(?P<dur_minute2>[0-9]+M(?:[0-9]+S)?)|(?P<dur_second2>[0-9]+S)))|(?P<dur_week>[0-9]+W))"),
+            ("email", r"(?P<local_part>(?P<dot_string>[^\s@\.]+(\.[^\s@\.]+)*))@((?P<domain>(?P<sub_domain>[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?)(\.(?P<sub_domain2>[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?))*)|\[(?P<ipv4>((([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9])))\.){3}(([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9]))))\])"),
+            ("hostname", r"[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*"),
+            ("ipv4", r"((([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9])))\.){3}(([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9])))"),
+            ("ipv6", r"(?:(?P<full>(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}))|(?:::(?:[0-9a-fA-F]{1,4}:){0,5}(?P<ls32>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:(?P<h16_1>[0-9a-fA-F]{1,4})?::(?:[0-9a-fA-F]{1,4}:){0,4}(?P<ls32_1>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,1}[0-9a-fA-F]{1,4})?::(?:[0-9a-fA-F]{1,4}:){0,3}(?P<ls32_2>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,2}[0-9a-fA-F]{1,4})?::(?:[0-9a-fA-F]{1,4}:){0,2}(?P<ls32_3>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,3}[0-9a-fA-F]{1,4})?::[0-9a-fA-F]{1,4}:(?P<ls32_4>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,4}[0-9a-fA-F]{1,4})?::(?P<ls32_5>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,5}[0-9a-fA-F]{1,4})?::(?P<h16_2>[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})?::)"),
+            ("uuid", r"(?P<time_low>[0-9a-fA-F]{8})-(?P<time_mid>[0-9a-fA-F]{4})-(?P<time_high_and_version>[0-9a-fA-F]{4})-(?P<clock_seq_and_reserved>[0-9a-fA-F]{2})(?P<clock_seq_low>[0-9a-fA-F]{2})-(?P<node>[0-9a-fA-F]{12})"),
+            ("unknown", r"(?s:.*)")
+        ])
     };
 }
 

From 4d89e818a014a0336826b12dd8bf4bc3143a7f0c Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 8 Nov 2024 12:51:30 -0800
Subject: [PATCH 07/73] initial port of numeric ranges

---
 parser/src/json/compiler.rs |  98 +++++++++-
 parser/src/json/mod.rs      |   1 +
 parser/src/json/numeric.rs  | 365 ++++++++++++++++++++++++++++++++++++
 3 files changed, 457 insertions(+), 7 deletions(-)
 create mode 100644 parser/src/json/numeric.rs

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index e7ab928..9970db5 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -5,6 +5,7 @@ use serde_json::{json, Value};
 use std::{collections::HashMap, vec};
 
 use super::formats::lookup_format;
+use super::numeric::{rx_float_range, rx_int_range};
 use crate::{
     api::{GrammarWithLexer, RegexSpec, TopLevelGrammar},
     GrammarBuilder, NodeRef,
@@ -63,6 +64,7 @@ const DEFS_KEYS: [&str; 4] = ["$defs", "definitions", "defs", "refs"];
 const ARRAY_KEYS: [&str; 4] = ["items", "prefixItems", "minItems", "maxItems"];
 const OBJECT_KEYS: [&str; 2] = ["properties", "additionalProperties"];
 const STRING_KEYS: [&str; 4] = ["minLength", "maxLength", "pattern", "format"];
+const NUMERIC_KEYS: [&str; 4] = ["minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum"];
 
 const CHAR_REGEX: &str = r#"(\\([\"\\\/bfnrt]|u[a-fA-F0-9]{4})|[^\"\\\x00-\x1F\x7F])"#;
 
@@ -83,7 +85,7 @@ fn validate_json_node_keys(node: &Value) -> Result<()> {
 
     for key in node.keys() {
         let key = &key.as_str();
-        if KEYWORDS.contains(key) || IGNORED_KEYS.contains(key) || DEFS_KEYS.contains(key) || ARRAY_KEYS.contains(key) || OBJECT_KEYS.contains(key) || STRING_KEYS.contains(key) {
+        if KEYWORDS.contains(key) || IGNORED_KEYS.contains(key) || DEFS_KEYS.contains(key) || ARRAY_KEYS.contains(key) || OBJECT_KEYS.contains(key) || STRING_KEYS.contains(key) || NUMERIC_KEYS.contains(key) {
             continue;
         }
         if key.starts_with("x-") || key.starts_with("$xsd-") {
@@ -135,6 +137,7 @@ fn mk_regex(rx: &str) -> RegexSpec {
 
 trait OptionalField {
     fn opt_u64(&self, key: &str) -> Result<Option<u64>>;
+    fn opt_f64(&self, key: &str) -> Result<Option<f64>>;
     fn opt_str(&self, key: &str) -> Result<Option<&str>>;
     fn opt_array(&self, key: &str) -> Result<Option<&Vec<Value>>>;
     #[allow(dead_code)]
@@ -162,6 +165,16 @@ impl OptionalField for Value {
         }
     }
 
+    fn opt_f64(&self, key: &str) -> Result<Option<f64>> {
+        if let Some(val) = self.get(key) {
+            val.as_f64()
+                .ok_or_else(|| expected_err(key, val, "float"))
+                .map(Some)
+        } else {
+            Ok(None)
+        }
+    }
+
     fn opt_str(&self, key: &str) -> Result<Option<&str>> {
         if let Some(val) = self.get(key) {
             val.as_str()
@@ -373,8 +386,53 @@ impl Compiler {
         match target_type_str {
             "null" => return Ok(self.builder.string("null")),
             "boolean" => return Ok(self.lexeme(r"true|false")),
-            "integer" => return Ok(self.json_int()),
-            "number" => return Ok(self.json_number()),
+            "integer" | "number" => {
+                let mut minimum = json_schema.opt_f64("minimum")?;
+                let mut maximum = json_schema.opt_f64("maximum")?;
+                let exclusive_minimum: bool = match json_schema.get("exclusiveMinimum") {
+                    // Draft4-style boolean
+                    Some(Value::Bool(b)) => *b,
+                    // Draft 2020-12 style number
+                    Some(Value::Number(n)) => {
+                        let n = n.as_f64()
+                            .ok_or_else(|| expected_err("exclusiveMinimum", &Value::Number(n.clone()), "unsigned integer"))?;
+                        match minimum {
+                            Some(min) if n < min => {
+                                false
+                            },
+                            _ => {
+                                minimum = Some(n);
+                                true
+                            }
+                        }
+                    },
+                    _ => false,
+                };
+                let exclusive_maximum: bool = match json_schema.get("exclusiveMaximum") {
+                    // Draft4-style boolean
+                    Some(Value::Bool(b)) => *b,
+                    // Draft 2020-12 style number
+                    Some(Value::Number(n)) => {
+                        let n = n.as_f64()
+                            .ok_or_else(|| expected_err("exclusiveMaximum", &Value::Number(n.clone()), "unsigned integer"))?;
+                        match maximum {
+                            Some(max) if n > max => {
+                                false
+                            },
+                            _ => {
+                                maximum = Some(n);
+                                true
+                            }
+                        }
+                    },
+                    _ => false,
+                };
+                if target_type_str == "integer" {
+                    return self.json_int(minimum, maximum, exclusive_minimum, exclusive_maximum);
+                } else {
+                    return self.json_number(minimum, maximum, exclusive_minimum, exclusive_maximum);
+                }
+            },
             "string" => {
                 let min_length = json_schema.opt_u64("minLength")?.unwrap_or(0);
                 let max_length = json_schema.opt_u64("maxLength")?;
@@ -411,12 +469,38 @@ impl Compiler {
         r
     }
 
-    fn json_int(&mut self) -> NodeRef {
-        self.lexeme(r"-?(?:0|[1-9][0-9]*)")
+    fn json_int(&mut self, minimum: Option<f64>, maximum: Option<f64>, exclusive_minimum: bool, exclusive_maximum: bool) -> Result<NodeRef> {
+        let minimum = match (minimum, exclusive_minimum) {
+            (Some(min_val), true) => {
+                if min_val.fract() != 0.0 {
+                    Some(min_val.ceil())
+                } else {
+                    Some(min_val + 1.0)
+                }
+            }
+            (Some(min_val), false) => Some(min_val.floor()),
+            _ => None,
+        }.map(|val| val as i64);
+        let maximum = match (maximum, exclusive_maximum) {
+            (Some(max_val), true) => {
+                if max_val.fract() != 0.0 {
+                    Some(max_val.floor())
+                } else {
+                    Some(max_val - 1.0)
+                }
+            },
+            (Some(max_val), false) => Some(max_val.ceil()),
+            _ => None,
+        }.map(|val| val as i64);
+        // TODO: handle errors in rx_int_range; currently it just panics
+        let rx = rx_int_range(minimum, maximum);
+        Ok(self.lexeme(&rx))
     }
 
-    fn json_number(&mut self) -> NodeRef {
-        self.lexeme(r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?")
+    fn json_number(&mut self, minimum: Option<f64>, maximum: Option<f64>, exclusive_minimum: bool, exclusive_maximum: bool) -> Result<NodeRef> {
+        // TODO: handle errors in rx_float_range; currently it just panics
+        let rx = rx_float_range(minimum, maximum, !exclusive_minimum, !exclusive_maximum);
+        Ok(self.lexeme(&rx))
     }
 
     fn json_simple_string(&mut self) -> NodeRef {
diff --git a/parser/src/json/mod.rs b/parser/src/json/mod.rs
index 3c89888..f40ea97 100644
--- a/parser/src/json/mod.rs
+++ b/parser/src/json/mod.rs
@@ -1,2 +1,3 @@
 pub mod compiler;
 mod formats;
+mod numeric;
diff --git a/parser/src/json/numeric.rs b/parser/src/json/numeric.rs
new file mode 100644
index 0000000..1f1fd6e
--- /dev/null
+++ b/parser/src/json/numeric.rs
@@ -0,0 +1,365 @@
+fn mk_or(parts: Vec<String>) -> String {
+    if parts.len() == 1 {
+        parts[0].clone()
+    } else {
+        format!("({})", parts.join("|"))
+    }
+}
+
+fn num_digits(n: i64) -> usize {
+    n.abs().to_string().len()
+}
+
+pub fn rx_int_range(left: Option<i64>, right: Option<i64>) -> String {
+    match (left, right) {
+        (None, None) => "-?(0|[1-9][0-9]*)".to_string(),
+        (Some(left), None) => {
+            if left < 0 {
+                mk_or(vec![
+                    rx_int_range(Some(left), Some(-1)),
+                    rx_int_range(Some(0), None),
+                ])
+            } else {
+                mk_or(vec![
+                    rx_int_range(
+                        Some(left),
+                        Some("9".repeat(num_digits(left)).parse::<i64>().unwrap()),
+                    ),
+                    format!("[1-9][0-9]{{{},}}", num_digits(left)),
+                ])
+            }
+        }
+        (None, Some(right)) => {
+            if right >= 0 {
+                mk_or(vec![
+                    rx_int_range(Some(0), Some(right)),
+                    rx_int_range(None, Some(-1)),
+                ])
+            } else {
+                format!("-{}", rx_int_range(Some(-right), None))
+            }
+        }
+        (Some(left), Some(right)) => {
+            assert!(left <= right);
+            if left < 0 {
+                if right < 0 {
+                    format!("(-{})", rx_int_range(Some(-right), Some(-left)))
+                } else {
+                    format!(
+                        "(-{}|{})",
+                        rx_int_range(Some(0), Some(-left)),
+                        rx_int_range(Some(0), Some(right))
+                    )
+                }
+            } else {
+                if num_digits(left) == num_digits(right) {
+                    let l = left.to_string();
+                    let r = right.to_string();
+                    if left == right {
+                        return format!("({})", l);
+                    }
+
+                    let lpref = &l[..l.len() - 1];
+                    let lx = &l[l.len() - 1..];
+                    let rpref = &r[..r.len() - 1];
+                    let rx = &r[r.len() - 1..];
+
+                    if lpref == rpref {
+                        return format!("({}{}[{}-{}])", lpref, "", lx, rx);
+                    }
+
+                    let left_rec: i64 = if lpref.is_empty() {
+                        0
+                    } else {
+                        lpref.parse().unwrap()
+                    };
+                    let right_rec: i64 = if rpref.is_empty() {
+                        0
+                    } else {
+                        rpref.parse().unwrap()
+                    };
+                    assert!(left_rec < right_rec);
+                    let mut parts = Vec::new();
+
+                    if lx != "0" {
+                        parts.push(format!("{}[{}-9]", lpref, lx));
+                    }
+
+                    if rx != "9" {
+                        parts.push(format!("{}[0-{}]", rpref, rx));
+                    }
+
+                    if left_rec + 1 <= right_rec - 1 {
+                        let inner = rx_int_range(Some(left_rec + 1), Some(right_rec - 1));
+                        parts.push(format!("{}[0-9]", inner));
+                    }
+
+                    mk_or(parts)
+                } else {
+                    let break_point = 10_i64.pow(num_digits(left) as u32) - 1;
+                    mk_or(vec![
+                        rx_int_range(Some(left), Some(break_point)),
+                        rx_int_range(Some(break_point + 1), Some(right)),
+                    ])
+                }
+            }
+        }
+    }
+}
+
+fn lexi_x_to_9(x: &str, incl: bool) -> String {
+    if incl {
+        if x.is_empty() {
+            "[0-9]*".to_string()
+        } else if x.len() == 1 {
+            format!("[{}-9][0-9]*", x)
+        } else {
+            let x0 = x.chars().next().unwrap().to_digit(10).unwrap();
+            let x_rest = &x[1..];
+            let mut parts = vec![format!(
+                "{}{}",
+                x.chars().next().unwrap(),
+                lexi_x_to_9(x_rest, incl)
+            )];
+            if x0 < 9 {
+                parts.push(format!("[{}-9][0-9]*", x0 + 1));
+            }
+            mk_or(parts)
+        }
+    } else {
+        if x.is_empty() {
+            "[0-9]*[1-9]".to_string()
+        } else {
+            let x0 = x.chars().next().unwrap().to_digit(10).unwrap();
+            let x_rest = &x[1..];
+            let mut parts = vec![format!(
+                "{}{}",
+                x.chars().next().unwrap(),
+                lexi_x_to_9(x_rest, incl)
+            )];
+            if x0 < 9 {
+                parts.push(format!("[{}-9][0-9]*", x0 + 1));
+            }
+            mk_or(parts)
+        }
+    }
+}
+
+fn lexi_0_to_x(x: &str, incl: bool) -> String {
+    if x.is_empty() {
+        assert!(incl);
+        "".to_string()
+    } else {
+        let x0 = x.chars().next().unwrap().to_digit(10).unwrap();
+        let x_rest = &x[1..];
+
+        if !incl && x.len() == 1 {
+            assert!(x0 > 0);
+            return format!("[0-{}][0-9]*", x0 - 1);
+        }
+
+        let mut parts = vec![format!(
+            "{}{}",
+            x.chars().next().unwrap(),
+            lexi_0_to_x(x_rest, incl)
+        )];
+        if x0 > 0 {
+            parts.push(format!("[0-{}][0-9]*", x0 - 1));
+        }
+        mk_or(parts)
+    }
+}
+
+fn lexi_range(ld: &str, rd: &str, ld_incl: bool, rd_incl: bool) -> String {
+    assert_eq!(ld.len(), rd.len());
+    if ld == rd {
+        assert!(ld_incl && rd_incl);
+        ld.to_string()
+    } else {
+        let l0 = ld.chars().next().unwrap().to_digit(10).unwrap();
+        let r0 = rd.chars().next().unwrap().to_digit(10).unwrap();
+        if l0 == r0 {
+            let ld_rest = &ld[1..];
+            let rd_rest = &rd[1..];
+            format!(
+                "{}{}",
+                ld.chars().next().unwrap(),
+                lexi_range(ld_rest, rd_rest, ld_incl, rd_incl)
+            )
+        } else {
+            assert!(l0 < r0);
+            let ld_rest = ld[1..].trim_end_matches('0');
+            let mut parts = vec![format!(
+                "{}{}",
+                ld.chars().next().unwrap(),
+                lexi_x_to_9(ld_rest, ld_incl)
+            )];
+            if l0 + 1 < r0 {
+                parts.push(format!("[{}-{}][0-9]*", l0 + 1, r0 - 1));
+            }
+            let rd_rest = rd[1..].trim_end_matches('0');
+            if !rd_rest.is_empty() || rd_incl {
+                parts.push(format!(
+                    "{}{}",
+                    rd.chars().next().unwrap(),
+                    lexi_0_to_x(rd_rest, rd_incl)
+                ));
+            }
+            mk_or(parts)
+        }
+    }
+}
+
+fn float_to_str(f: f64) -> String {
+    format!("{}", f)
+}
+
+pub fn rx_float_range(
+    left: Option<f64>,
+    right: Option<f64>,
+    left_inclusive: bool,
+    right_inclusive: bool,
+) -> String {
+    match (left, right) {
+        (None, None) => "-?(0|[1-9][0-9]*)(\\.[0-9]+)?([eE][+-]?[0-9]+)?".to_string(),
+        (Some(left), None) => {
+            if left < 0.0 {
+                mk_or(vec![
+                    rx_float_range(Some(left), Some(0.0), left_inclusive, false),
+                    rx_float_range(Some(0.0), None, true, false),
+                ])
+            } else {
+                let left_int_part = left as i64;
+                mk_or(vec![
+                    rx_float_range(
+                        Some(left),
+                        Some(10f64.powi(num_digits(left_int_part) as i32)),
+                        left_inclusive,
+                        false,
+                    ),
+                    format!(
+                        "[1-9][0-9]{{{},}}(\\.[0-9]+)?",
+                        num_digits(left_int_part)
+                    ),
+                ])
+            }
+        }
+        (None, Some(right)) => {
+            if right == 0.0 {
+                let r = format!("-{}", rx_float_range(Some(0.0), None, false, false));
+                if right_inclusive {
+                    mk_or(vec![r, "0".to_string()])
+                } else {
+                    r
+                }
+            } else if right > 0.0 {
+                mk_or(vec![
+                    format!("-{}", rx_float_range(Some(0.0), None, false, false)),
+                    rx_float_range(Some(0.0), Some(right), true, right_inclusive),
+                ])
+            } else {
+                format!("-{}", rx_float_range(Some(-right), None, right_inclusive, false))
+            }
+        }
+        (Some(left), Some(right)) => {
+            assert!(left <= right);
+            if left == right {
+                if left_inclusive && right_inclusive {
+                    format!("({})", float_to_str(left))
+                } else {
+                    panic!("Empty range");
+                }
+            } else if left < 0.0 {
+                if right < 0.0 {
+                    format!(
+                        "(-{})",
+                        rx_float_range(
+                            Some(-right),
+                            Some(-left),
+                            right_inclusive,
+                            left_inclusive
+                        )
+                    )
+                } else {
+                    let mut parts = vec![];
+                    let neg_part = rx_float_range(
+                        Some(0.0),
+                        Some(-left),
+                        false, // we don't allow -0
+                        left_inclusive,
+                    );
+                    parts.push(format!("(-{})", neg_part));
+
+                    if right > 0.0 || right_inclusive {
+                        let pos_part = rx_float_range(
+                            Some(0.0),
+                            Some(right),
+                            true,  // always include 0
+                            right_inclusive,
+                        );
+                        parts.push(pos_part);
+                    }
+                    mk_or(parts)
+                }
+            } else {
+                let l = float_to_str(left);
+                let r = float_to_str(right);
+                assert!(l != r);
+                assert!(!l.contains('e') && !r.contains('e'));
+
+                if !left.is_finite() || !right.is_finite() {
+                    panic!("Infinite numbers not supported");
+                }
+
+                let mut left_rec: i64 = l.split('.').next().unwrap().parse().unwrap();
+                let right_rec: i64 = r.split('.').next().unwrap().parse().unwrap();
+
+                let mut ld = l.split('.').nth(1).unwrap_or("").to_string();
+                let mut rd = r.split('.').nth(1).unwrap_or("").to_string();
+
+                if left_rec == right_rec {
+                    while ld.len() < rd.len() {
+                        ld.push('0');
+                    }
+                    while rd.len() < ld.len() {
+                        rd.push('0');
+                    }
+                    let suff = format!("\\.{}", lexi_range(&ld, &rd, left_inclusive, right_inclusive));
+                    if ld.parse::<i64>().unwrap() == 0 {
+                        format!("({}({})?)", left_rec, suff)
+                    } else {
+                        format!("({}{})", left_rec, suff)
+                    }
+                } else {
+                    let mut parts = vec![];
+                    if !ld.is_empty() || !left_inclusive {
+                        parts.push(format!(
+                            "({}\\.{})",
+                            left_rec,
+                            lexi_x_to_9(&ld, left_inclusive)
+                        ));
+                        left_rec += 1;
+                    }
+
+                    if right_rec - 1 >= left_rec {
+                        let inner = rx_int_range(Some(left_rec), Some(right_rec - 1));
+                        parts.push(format!("({}(\\.[0-9]+)?)", inner));
+                    }
+
+                    if !rd.is_empty() || right_inclusive {
+                        parts.push(format!(
+                            "({}{}{})",
+                            right_rec,
+                            if !rd.is_empty() { "\\." } else { "" },
+                            lexi_0_to_x(&rd, right_inclusive)
+                        ));
+                    } else if right_inclusive {
+                        parts.push(format!("{}(\\.0+)?", right_rec));
+                    }
+
+                    mk_or(parts)
+                }
+            }
+        }
+    }
+}

From 9d58157c5dd6ccbb52044b85e90e217e024ac872 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 8 Nov 2024 13:11:42 -0800
Subject: [PATCH 08/73] fix right inclusivity

---
 parser/src/json/numeric.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/parser/src/json/numeric.rs b/parser/src/json/numeric.rs
index 1f1fd6e..725b197 100644
--- a/parser/src/json/numeric.rs
+++ b/parser/src/json/numeric.rs
@@ -346,11 +346,10 @@ pub fn rx_float_range(
                         parts.push(format!("({}(\\.[0-9]+)?)", inner));
                     }
 
-                    if !rd.is_empty() || right_inclusive {
+                    if !rd.is_empty() {
                         parts.push(format!(
-                            "({}{}{})",
+                            "({}(\\.{})?)",
                             right_rec,
-                            if !rd.is_empty() { "\\." } else { "" },
                             lexi_0_to_x(&rd, right_inclusive)
                         ));
                     } else if right_inclusive {

From c777deb9ed3bde575457a84ab20297f5f52e2c0d Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 8 Nov 2024 13:16:23 -0800
Subject: [PATCH 09/73] fix integer bounds from floats

---
 parser/src/json/compiler.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index 9970db5..b3f17fc 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -478,7 +478,7 @@ impl Compiler {
                     Some(min_val + 1.0)
                 }
             }
-            (Some(min_val), false) => Some(min_val.floor()),
+            (Some(min_val), false) => Some(min_val.ceil()),
             _ => None,
         }.map(|val| val as i64);
         let maximum = match (maximum, exclusive_maximum) {
@@ -489,7 +489,7 @@ impl Compiler {
                     Some(max_val - 1.0)
                 }
             },
-            (Some(max_val), false) => Some(max_val.ceil()),
+            (Some(max_val), false) => Some(max_val.floor()),
             _ => None,
         }.map(|val| val as i64);
         // TODO: handle errors in rx_int_range; currently it just panics

From 1c6be002636afcb006c9c0a6fc8d43e7748d3364 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 8 Nov 2024 13:45:41 -0800
Subject: [PATCH 10/73] more fixes for ints

---
 parser/src/json/numeric.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/parser/src/json/numeric.rs b/parser/src/json/numeric.rs
index 725b197..75c14ef 100644
--- a/parser/src/json/numeric.rs
+++ b/parser/src/json/numeric.rs
@@ -89,8 +89,8 @@ pub fn rx_int_range(left: Option<i64>, right: Option<i64>) -> String {
                         parts.push(format!("{}[0-{}]", rpref, rx));
                     }
 
-                    if left_rec + 1 <= right_rec - 1 {
-                        let inner = rx_int_range(Some(left_rec + 1), Some(right_rec - 1));
+                    if left_rec <= right_rec {
+                        let inner = rx_int_range(Some(left_rec), Some(right_rec));
                         parts.push(format!("{}[0-9]", inner));
                     }
 

From a85000ffe9e7a4d386a09ae06a8b5e5a3ced50ca Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 8 Nov 2024 14:23:28 -0800
Subject: [PATCH 11/73] whitespace_flexible and separators

---
 parser/src/earley/from_guidance.rs |  2 +-
 parser/src/ffi.rs                  |  2 +-
 parser/src/json/compiler.rs        | 29 +++++++++++++++++++----------
 rust/src/py.rs                     | 21 ++++++++++++++++-----
 4 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/parser/src/earley/from_guidance.rs b/parser/src/earley/from_guidance.rs
index f681963..031979b 100644
--- a/parser/src/earley/from_guidance.rs
+++ b/parser/src/earley/from_guidance.rs
@@ -98,7 +98,7 @@ fn grammar_from_json(
                 input.lark_grammar.is_none(),
                 "cannot have both json_schema and lark_grammar"
             );
-            let opts = JsonCompileOptions { compact: false };
+            let opts = JsonCompileOptions::default();
             opts.json_to_llg_no_validate(json_schema)?
         } else {
             let items = parse_lark(input.lark_grammar.as_ref().unwrap())?;
diff --git a/parser/src/ffi.rs b/parser/src/ffi.rs
index e29ec5d..b37264b 100644
--- a/parser/src/ffi.rs
+++ b/parser/src/ffi.rs
@@ -288,7 +288,7 @@ fn new_constraint_json(init: &LlgConstraintInit, json_schema: *const c_char) ->
         .map_err(|_| anyhow::anyhow!("Invalid UTF-8 in json_schema"))?;
     let json_schema = serde_json::from_str(json_schema)
         .map_err(|e| anyhow::anyhow!("Invalid JSON in json_schema: {e}"))?;
-    let opts = JsonCompileOptions { compact: false };
+    let opts = JsonCompileOptions::default();
     let grammar = opts
         .json_to_llg_no_validate(&json_schema)
         .map_err(|e| anyhow::anyhow!("Error compiling JSON schema to LLG: {e}"))?;
diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index b3f17fc..358caf0 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -17,7 +17,9 @@ use crate::{
 
 #[derive(Debug, Default, Clone)]
 pub struct JsonCompileOptions {
-    pub compact: bool,
+    pub item_separator: String,
+    pub key_separator: String,
+    pub whitespace_flexible: bool,
 }
 
 fn to_compact_json(target: &serde_json::Value) -> String {
@@ -117,6 +119,13 @@ macro_rules! cache {
 }
 
 impl JsonCompileOptions {
+    pub fn default() -> Self {
+        Self {
+            item_separator: ",".to_string(),
+            key_separator: ":".to_string(),
+            whitespace_flexible: true,
+        }
+    }
     pub fn json_to_llg(&self, schema: &Value) -> Result<TopLevelGrammar> {
         let mut compiler = Compiler::new(self.clone());
         compiler.validate(schema)?;
@@ -258,10 +267,10 @@ impl Compiler {
 
     pub fn execute(&mut self, schema: &Value) -> Result<()> {
         self.builder.add_grammar(GrammarWithLexer {
-            greedy_skip_rx: if self.options.compact {
-                None
-            } else {
+            greedy_skip_rx: if self.options.whitespace_flexible {
                 Some(mk_regex(r"[\x20\x0A\x0D\x09]+"))
+            } else {
+                None
             },
             ..GrammarWithLexer::default()
         });
@@ -553,7 +562,7 @@ impl Compiler {
         if !properties.is_empty() {
             grammars.extend(self.process_properties(properties)?);
             if additional_properties != &Value::Bool(false) {
-                grammars.push(self.builder.string(","));
+                grammars.push(self.builder.string(&self.options.item_separator));
             }
         }
 
@@ -574,11 +583,11 @@ impl Compiler {
 
         for (name, property_schema) in properties {
             result.push(self.builder.string(&format!("\"{}\"", name)));
-            result.push(self.builder.string(":"));
+            result.push(self.builder.string(&self.options.key_separator));
             result.push(self.gen_json(property_schema)?);
             properties_added += 1;
             if properties_added < properties.len() {
-                result.push(self.builder.string(","));
+                result.push(self.builder.string(&self.options.item_separator));
             }
         }
 
@@ -587,7 +596,7 @@ impl Compiler {
 
     fn process_additional_properties(&mut self, additional_properties: &Value) -> Result<NodeRef> {
         let str = self.json_simple_string();
-        let colon = self.builder.string(":");
+        let colon = self.builder.string(&self.options.key_separator);
         let the_rest = self.gen_json(additional_properties)?;
         let item = self.builder.join(&[str, colon, the_rest]);
         let inner = self.sequence(item);
@@ -595,7 +604,7 @@ impl Compiler {
     }
 
     fn sequence(&mut self, item: NodeRef) -> NodeRef {
-        let comma = self.builder.string(",");
+        let comma = self.builder.string(&self.options.item_separator);
         let item_comma = self.builder.join(&[item, comma]);
         let item_comma_star = self.builder.zero_or_more(item_comma);
         self.builder.join(&[item_comma_star, item])
@@ -719,7 +728,7 @@ impl Compiler {
         }
 
         let mut grammars: Vec<NodeRef> = vec![self.builder.string("[")];
-        let comma = self.builder.string(",");
+        let comma = self.builder.string(&self.options.item_separator);
 
         if !required_items.is_empty() {
             grammars.push(required_items[0]);
diff --git a/rust/src/py.rs b/rust/src/py.rs
index a0d38b6..4abf1a8 100644
--- a/rust/src/py.rs
+++ b/rust/src/py.rs
@@ -244,22 +244,33 @@ impl TokenizerEnv for LLTokenizer {
 #[derive(Clone)]
 #[pyclass]
 struct JsonCompiler {
-    #[pyo3(get, set)]
-    compact: bool,
+    item_separator: String,
+    key_separator: String,
+    whitespace_flexible: bool
 }
 
 #[pymethods]
 impl JsonCompiler {
     #[new]
-    fn py_new(compact: Option<bool>) -> Self {
+    #[pyo3(signature = (separators = None, whitespace_flexible = false))]
+    fn py_new(separators: Option<(String, String)>, whitespace_flexible: bool) -> Self {
+        let (item_separator, key_separator) = separators.unwrap_or_else(|| if whitespace_flexible {
+            (",".to_owned(), ":".to_owned())
+        } else {
+            (", ".to_owned(), ": ".to_owned())
+        });
         JsonCompiler {
-            compact: compact.unwrap_or(false),
+            item_separator: item_separator,
+            key_separator: key_separator,
+            whitespace_flexible,
         }
     }
     fn compile(&self, schema: &str) -> PyResult<String> {
         let schema: Value = serde_json::from_str(schema).map_err(val_error)?;
         let compile_options = JsonCompileOptions {
-            compact: self.compact,
+            item_separator: self.item_separator.clone(),
+            key_separator: self.key_separator.clone(),
+            whitespace_flexible: self.whitespace_flexible,
         };
         let tlg = compile_options.json_to_llg(&schema).map_err(val_error)?;
         let grammar = &tlg.grammars[0];

From ba817e412a0977fcb92fde599c9f1fee0346eddb Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 8 Nov 2024 15:08:32 -0800
Subject: [PATCH 12/73] whitespace flexible consts and enums

---
 parser/src/json/compiler.rs | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index 358caf0..89cda5d 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -354,16 +354,15 @@ impl Compiler {
 
         // Process const
         if let Some(const_value) = json_schema.get("const") {
-            let compact_const = to_compact_json(const_value);
-            return Ok(self.builder.string(&compact_const));
+            return self.gen_json_const(const_value);
         }
 
         // Process enum
         if let Some(enum_array) = json_schema.opt_array("enum")? {
             let options = enum_array
                 .iter()
-                .map(|opt| self.builder.string(&to_compact_json(opt)))
-                .collect::<Vec<_>>();
+                .map(|opt| self.gen_json_const(opt))
+                .collect::<Result<Vec<_>, _>>()?;
             return Ok(self.builder.select(&options));
         }
 
@@ -391,6 +390,30 @@ impl Compiler {
         Ok(self.builder.select(&nodes))
     }
 
+    fn gen_json_const(&mut self, const_value: &Value) -> Result<NodeRef> {
+        // Recursively build a grammar for a constant value (just to play nicely with separators and whitespace flexibility)
+        match const_value {
+            Value::Object(values) => {
+                let properties = values.iter()
+                    .map(|(k, v)| {
+                        (k.clone(), json!({"const": v}))
+                    })
+                    .collect::<serde_json::Map<_, _>>();
+                self.gen_json_object(&properties, &Value::Bool(false))
+            },
+            Value::Array(values) => {
+                let n_items = values.len() as u64;
+                let prefix_items = values.iter().map(|v| json!({"const": v})).collect::<Vec<_>>();
+                self.gen_json_array(&prefix_items, &Value::Bool(false), n_items, Some(n_items))
+            },
+            _ => {
+                // let serde_json dump simple values
+                let const_str = to_compact_json(const_value);
+                Ok(self.builder.string(&const_str))
+            }
+        }
+    }
+
     fn gen_json_type(&mut self, target_type_str: &str, json_schema: &Value) -> Result<NodeRef> {
         match target_type_str {
             "null" => return Ok(self.builder.string("null")),

From 8bbd244bb8a8a707b9f1853f12a4a867def562ba Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 8 Nov 2024 18:27:12 -0800
Subject: [PATCH 13/73] required

---
 parser/src/json/compiler.rs | 124 ++++++++++++++++++++++++------------
 1 file changed, 83 insertions(+), 41 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index 89cda5d..1982305 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -64,7 +64,7 @@ const IGNORED_KEYS: [&str; 19] = [
 const DEFS_KEYS: [&str; 4] = ["$defs", "definitions", "defs", "refs"];
 
 const ARRAY_KEYS: [&str; 4] = ["items", "prefixItems", "minItems", "maxItems"];
-const OBJECT_KEYS: [&str; 2] = ["properties", "additionalProperties"];
+const OBJECT_KEYS: [&str; 3] = ["properties", "additionalProperties", "required"];
 const STRING_KEYS: [&str; 4] = ["minLength", "maxLength", "pattern", "format"];
 const NUMERIC_KEYS: [&str; 4] = ["minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum"];
 
@@ -399,7 +399,7 @@ impl Compiler {
                         (k.clone(), json!({"const": v}))
                     })
                     .collect::<serde_json::Map<_, _>>();
-                self.gen_json_object(&properties, &Value::Bool(false))
+                self.gen_json_object(&properties, &Value::Bool(false), properties.keys().cloned().collect())
             },
             Value::Array(values) => {
                 let n_items = values.len() as u64;
@@ -486,7 +486,18 @@ impl Compiler {
                 let additional_properties = json_schema
                     .get("additionalProperties")
                     .unwrap_or(&Value::Bool(true));
-                return self.gen_json_object(properties, additional_properties);
+                let required: Vec<String> = json_schema.opt_array("required")?
+                    .map(|values| {
+                        values.iter()
+                            .map(|v| {
+                                v.as_str()
+                                    .map(|s| s.to_string()) // Convert &str to String
+                                    .ok_or_else(|| expected_err("required", v, "array of string"))
+                            })
+                            .collect::<Result<Vec<_>, _>>()
+                    })
+                    .unwrap_or_else(|| Ok(vec![]))?;
+                return self.gen_json_object(properties, additional_properties, required);
             }
             _ => bail!("Unsupported type in schema: {}", target_type_str),
         }
@@ -579,51 +590,82 @@ impl Compiler {
         &mut self,
         properties: &serde_json::Map<String, Value>,
         additional_properties: &Value,
+        required: Vec<String>,
     ) -> Result<NodeRef> {
-        let mut grammars: Vec<NodeRef> = vec![self.builder.string("{")];
-
-        if !properties.is_empty() {
-            grammars.extend(self.process_properties(properties)?);
-            if additional_properties != &Value::Bool(false) {
-                grammars.push(self.builder.string(&self.options.item_separator));
+        let mut illegal_keys: Vec<String> = vec![];
+        let mut items: Vec<(NodeRef, bool)> = vec![];
+        for name in properties.keys().chain(required.iter().filter(|n| !properties.contains_key(n.as_str()))) {
+            let property_schema = properties.get(name).unwrap_or(additional_properties);
+            let is_required = required.contains(name);
+            if property_schema == &Value::Bool(false) {
+                if is_required {
+                    bail!("Required property has 'false' schema: {}", name);
+                }
+                illegal_keys.push(name.clone());
+                continue;
             }
+            let name = self.builder.string(&format!("\"{}\"", name));
+            let colon = self.builder.string(&self.options.key_separator);
+            let the_rest = self.gen_json(property_schema)?;
+            let item = self.builder.join(&[name, colon, the_rest]);
+            items.push((item, is_required));
         }
-
         if additional_properties != &Value::Bool(false) {
-            grammars.push(self.process_additional_properties(additional_properties)?);
-        }
-
-        grammars.push(self.builder.string("}"));
-        Ok(self.builder.join(&grammars))
+            // TODO: illegal keys and NOT(properties keys)
+            let name = self.json_simple_string();
+            let colon = self.builder.string(&self.options.key_separator);
+            let the_rest = self.gen_json(additional_properties)?;
+            let item = self.builder.join(&[name, colon, the_rest]);
+            let seq = self.sequence(item);
+            items.push((seq, false));
+        }
+        let opener = self.builder.string("{");
+        let inner = self.ordered_sequence(&items, false);
+        let closer = self.builder.string("}");
+        Ok(self.builder.join(&[opener, inner, closer]))
     }
 
-    fn process_properties(
-        &mut self,
-        properties: &serde_json::Map<String, Value>,
-    ) -> Result<Vec<NodeRef>> {
-        let mut result = vec![];
-        let mut properties_added = 0;
-
-        for (name, property_schema) in properties {
-            result.push(self.builder.string(&format!("\"{}\"", name)));
-            result.push(self.builder.string(&self.options.key_separator));
-            result.push(self.gen_json(property_schema)?);
-            properties_added += 1;
-            if properties_added < properties.len() {
-                result.push(self.builder.string(&self.options.item_separator));
-            }
-        }
-
-        Ok(result)
-    }
 
-    fn process_additional_properties(&mut self, additional_properties: &Value) -> Result<NodeRef> {
-        let str = self.json_simple_string();
-        let colon = self.builder.string(&self.options.key_separator);
-        let the_rest = self.gen_json(additional_properties)?;
-        let item = self.builder.join(&[str, colon, the_rest]);
-        let inner = self.sequence(item);
-        Ok(self.builder.optional(inner))
+    fn ordered_sequence(&mut self, items: &[(NodeRef, bool)], prefixed: bool) -> NodeRef {
+        if items.is_empty() {
+            return self.builder.string("");
+        }
+        let comma = self.builder.string(&self.options.item_separator);
+        let (item, required) = items[0];
+        let rest = &items[1..];
+
+        match (prefixed, required) {
+            (true, true) => {
+                // If we know we have preceeding elements, we can safely just add a (',' + e)
+                let rest_seq = self.ordered_sequence(rest, true);
+                self.builder.join(&[comma, item, rest_seq])
+            },
+            (true, false) => {
+                // If we know we have preceeding elements, we can safely just add an optional(',' + e)
+                // TODO optimization: if the rest is all optional, we can nest the rest in the optional
+                let comma_item = self.builder.join(&[comma, item]);
+                let optional_comma_item = self.builder.optional(comma_item);
+                let rest_seq = self.ordered_sequence(rest, true);
+                self.builder.join(&[optional_comma_item, rest_seq])
+            },
+            (false, true) => {
+                // No preceeding elements, so we just add the element (no comma)
+                let rest_seq = self.ordered_sequence(rest, true);
+                self.builder.join(&[item, rest_seq])
+            },
+            (false, false) => {
+                // No preceeding elements, but our element is optional. If we add the element, the remaining
+                // will be prefixed, else they are not.
+                // TODO: same nested optimization as above
+                let prefixed_rest = self.ordered_sequence(rest, true);
+                let unprefixed_rest = self.ordered_sequence(rest, false);
+                let opts = [
+                    self.builder.join(&[item, prefixed_rest]),
+                    unprefixed_rest,
+                ];
+                self.builder.select(&opts)
+            },
+        }
     }
 
     fn sequence(&mut self, item: NodeRef) -> NodeRef {

From f85598a5e9befc053da8d0a69ecbec30a8a58777 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Sat, 9 Nov 2024 20:20:19 -0800
Subject: [PATCH 14/73] types is wonky

---
 parser/src/json/compiler.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index a51b383..c7af5fa 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -346,11 +346,10 @@ impl Compiler {
             return self.gen_json_type(tp, json_schema)
         }
 
+        let tps = TYPES.iter().map(|s| Value::String(s.to_string())).collect::<Vec<_>>();
         let types = match json_schema.opt_array("type")? {
             Some(types) => types,
-            None => {
-                &TYPES.iter().map(|s| Value::String(s.to_string())).collect::<Vec<_>>()
-            }
+            None => &tps,
         };
 
         let nodes = types

From 9aa5db5b94fa25e038fc3cdc74dfc64f129e07fe Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Sat, 9 Nov 2024 20:20:45 -0800
Subject: [PATCH 15/73] substring (may pull out into separate PR)

---
 parser/src/json/mod.rs       |   1 +
 parser/src/json/substring.rs | 154 +++++++++++++++++++++++++++++++++++
 2 files changed, 155 insertions(+)
 create mode 100644 parser/src/json/substring.rs

diff --git a/parser/src/json/mod.rs b/parser/src/json/mod.rs
index f40ea97..832988d 100644
--- a/parser/src/json/mod.rs
+++ b/parser/src/json/mod.rs
@@ -1,3 +1,4 @@
 pub mod compiler;
 mod formats;
 mod numeric;
+mod substring;
diff --git a/parser/src/json/substring.rs b/parser/src/json/substring.rs
new file mode 100644
index 0000000..42d5e9d
--- /dev/null
+++ b/parser/src/json/substring.rs
@@ -0,0 +1,154 @@
+use std::{collections::HashMap, vec};
+use derivre::{ExprRef, Regex, RegexAst, RegexBuilder};
+
+#[derive(Debug)]
+struct State {
+    len: usize,
+    link: Option<usize>,
+    next: HashMap<char, usize>,
+}
+
+/// For details see https://en.wikipedia.org/wiki/Suffix_automaton.
+/// Implementation is based on https://cp-algorithms.com/string/suffix-automaton.html
+struct SuffixAutomaton {
+    states: Vec<State>,
+    last: usize,
+}
+
+impl SuffixAutomaton {
+    fn new() -> Self {
+        let init_state = State {
+            len: 0,
+            link: None,
+            next: HashMap::new(),
+        };
+        SuffixAutomaton {
+            states: vec![init_state],
+            last: 0,
+        }
+    }
+
+    fn from_string(s: &str) -> Self {
+        let mut sa = SuffixAutomaton::new();
+        for c in s.chars() {
+            sa.extend(c);
+        }
+        sa
+    }
+
+    fn extend(&mut self, c: char) {
+        let cur_index = self.states.len();
+        self.states.push(
+            State {
+                len: self.states[self.last].len + 1,
+                link: None,
+                next: HashMap::new(),
+            }
+        );
+
+        let mut p = Some(self.last);
+        while let Some(pp) = p{
+            if self.states[pp].next.contains_key(&c) {
+                break;
+            }
+            self.states[pp].next.insert(c, cur_index);
+            p = self.states[pp].link;
+        }
+
+        if let Some(pp) = p {
+            let q = self.states[pp].next[&c];
+            if self.states[pp].len + 1 == self.states[q].len {
+                self.states[cur_index].link = Some(q);
+            } else {
+                let clone_index = self.states.len();
+                self.states.push(
+                    State {
+                        len: self.states[pp].len + 1,
+                        link: self.states[q].link,
+                        next: self.states[q].next.clone(),
+                    }
+                );
+                while let Some(ppp) = p {
+                    if self.states[ppp].next[&c] == q {
+                        self.states[ppp].next.insert(c, clone_index);
+                    } else {
+                        break;
+                    }
+                    p = self.states[ppp].link;
+                }
+                self.states[q].link = Some(clone_index);
+                self.states[cur_index].link = Some(clone_index);
+            }
+        } else {
+            self.states[cur_index].link = Some(0);
+        }
+        self.last = cur_index;
+    }
+}
+
+pub fn substring(string: &str) -> Result<Regex, anyhow::Error> {
+    let sa = SuffixAutomaton::from_string(string);
+    let mut builder = RegexBuilder::new();
+    let mut state_stack = vec![0];
+    let mut node_cache: HashMap<usize, ExprRef> = HashMap::new();
+
+    while let Some(state_index) = state_stack.last() {
+        let state = &sa.states[*state_index];
+        if node_cache.contains_key(state_index) {
+            state_stack.pop();
+            continue;
+        }
+
+        if state.next.is_empty() {
+            let expr = builder.mk(&RegexAst::EmptyString)?;
+            node_cache.insert(*state_index, expr);
+            state_stack.pop();
+            continue;
+        }
+
+        let unprocessed_children = state.next
+            .values()
+            .filter(|child_index| !node_cache.contains_key(child_index))
+            .collect::<Vec<_>>();
+        if !unprocessed_children.is_empty() {
+            state_stack.extend(unprocessed_children);
+            continue;
+        }
+
+        let options = state.next
+            .keys()
+            .map(|c| {
+                RegexAst::Concat(vec![
+                    RegexAst::Literal(c.to_string()),
+                    RegexAst::ExprRef(node_cache[&state.next[c]]),
+                ])
+            })
+            .collect::<Vec<_>>();
+        let node = RegexAst::Or([options, vec![RegexAst::EmptyString]].concat());
+        let expr = builder.mk(&node)?;
+        node_cache.insert(*state_index, expr);
+        state_stack.pop();
+    }
+    Ok(builder.to_regex(node_cache[&0]))
+}
+
+mod tests {
+    use super::{substring, SuffixAutomaton};
+
+    #[test]
+    fn test_suffix_automaton() {
+        let sa = SuffixAutomaton::from_string("abacaba");
+        println!("{:?}", sa.states);
+    }
+
+    #[test]
+    fn test_substring() {
+        let regex = substring("abacaba").unwrap();
+        assert_eq!(regex.clone().is_match("acab"), true);
+        assert_eq!(regex.clone().is_match("abaca"), true);
+        assert_eq!(regex.clone().is_match("acaba"), true);
+        assert_eq!(regex.clone().is_match("abacaba"), true);
+        assert_eq!(regex.clone().is_match("acabab"), false);
+        assert_eq!(regex.clone().is_match("cabaca"), false);
+    }
+}

From 49d2ca6fa31c8d41d34a17477bf915273500493e Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Sat, 9 Nov 2024 20:24:24 -0800
Subject: [PATCH 16/73] drop non-test

---
 parser/src/json/substring.rs | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/parser/src/json/substring.rs b/parser/src/json/substring.rs
index 42d5e9d..083ebea 100644
--- a/parser/src/json/substring.rs
+++ b/parser/src/json/substring.rs
@@ -135,12 +135,6 @@ pub fn substring(string: &str) -> Result<Regex, anyhow::Error> {
 mod tests {
     use super::{substring, SuffixAutomaton};
 
-    #[test]
-    fn test_suffix_automaton() {
-        let sa = SuffixAutomaton::from_string("abacaba");
-        println!("{:?}", sa.states);
-    }
-
     #[test]
     fn test_substring() {
         let regex = substring("abacaba").unwrap();

From 729425f1d599fa816f3de28fc24f52d819300aa4 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Sat, 9 Nov 2024 20:31:07 -0800
Subject: [PATCH 17/73] clean up

---
 parser/src/json/substring.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/parser/src/json/substring.rs b/parser/src/json/substring.rs
index 083ebea..90bc36b 100644
--- a/parser/src/json/substring.rs
+++ b/parser/src/json/substring.rs
@@ -115,7 +115,7 @@ pub fn substring(string: &str) -> Result<Regex, anyhow::Error> {
             continue;
         }
 
-        let options = state.next
+        let mut options = state.next
             .keys()
             .map(|c| {
                 RegexAst::Concat(vec![
@@ -124,7 +124,8 @@ pub fn substring(string: &str) -> Result<Regex, anyhow::Error> {
                 ])
             })
             .collect::<Vec<_>>();
-        let node = RegexAst::Or([options, vec![RegexAst::EmptyString]].concat());
+        options.push(RegexAst::EmptyString);
+        let node = RegexAst::Or(options);
         let expr = builder.mk(&node)?;
         node_cache.insert(*state_index, expr);
         state_stack.pop();
@@ -133,7 +134,7 @@ pub fn substring(string: &str) -> Result<Regex, anyhow::Error> {
 }
 
 mod tests {
-    use super::{substring, SuffixAutomaton};
+    use super::substring;
 
     #[test]
     fn test_substring() {

From 46bad35d74ea482d7101d4ed33ce28a75b311176 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Sat, 9 Nov 2024 20:35:49 -0800
Subject: [PATCH 18/73] null test

---
 parser/src/json/substring.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/parser/src/json/substring.rs b/parser/src/json/substring.rs
index 90bc36b..a1f29a6 100644
--- a/parser/src/json/substring.rs
+++ b/parser/src/json/substring.rs
@@ -139,6 +139,7 @@ mod tests {
     #[test]
     fn test_substring() {
         let regex = substring("abacaba").unwrap();
+        assert_eq!(regex.clone().is_match(""), true);
         assert_eq!(regex.clone().is_match("acab"), true);
         assert_eq!(regex.clone().is_match("abaca"), true);
         assert_eq!(regex.clone().is_match("acaba"), true);

From a898bf342c2593b58bfadcfc75bcac521af35bdd Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Sun, 10 Nov 2024 19:59:42 -0800
Subject: [PATCH 19/73] impl Default for JsonCompileOptions

---
 parser/src/json/compiler.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index c7af5fa..0f35a95 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -13,7 +13,7 @@ use crate::{
 // TODO: array maxItems etc limits
 // TODO: schemastore/src/schemas/json/BizTalkServerApplicationSchema.json - this breaks 1M fuel on lexer, why?!
 
-#[derive(Debug, Default, Clone)]
+#[derive(Debug, Clone)]
 pub struct JsonCompileOptions {
     pub item_separator: String,
     pub key_separator: String,
@@ -116,14 +116,17 @@ macro_rules! cache {
     };
 }
 
-impl JsonCompileOptions {
-    pub fn default() -> Self {
+impl Default for JsonCompileOptions {
+    fn default() -> Self {
         Self {
             item_separator: ",".to_string(),
             key_separator: ":".to_string(),
             whitespace_flexible: true,
         }
     }
+}
+
+impl JsonCompileOptions {
     pub fn json_to_llg(&self, schema: &Value) -> Result<TopLevelGrammar> {
         let mut compiler = Compiler::new(self.clone());
         #[cfg(feature = "jsonschema_validation")]

From 49b8b6b9245c76f7f29a40f4fc2fb582a814aeff Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Mon, 11 Nov 2024 15:49:23 -0800
Subject: [PATCH 20/73] taken names

---
 parser/src/grammar_builder.rs |  8 ++++++++
 parser/src/json/compiler.rs   | 29 +++++++++++++++++++++++------
 2 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/parser/src/grammar_builder.rs b/parser/src/grammar_builder.rs
index 1ab2d38..9ad835f 100644
--- a/parser/src/grammar_builder.rs
+++ b/parser/src/grammar_builder.rs
@@ -90,6 +90,14 @@ impl RegexBuilder {
         self.add_node(RegexNode::Repeat(node, min, max))
     }
 
+    pub fn not(&mut self, node: RegexId) -> RegexId {
+        self.add_node(RegexNode::Not(node))
+    }
+
+    pub fn and(&mut self, nodes: Vec<RegexId>) -> RegexId {
+        self.add_node(RegexNode::And(nodes))
+    }
+
     fn finalize(&mut self) -> Vec<RegexNode> {
         let r = std::mem::take(&mut self.nodes);
         *self = Self::new();
diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index 0f35a95..f1c8f3f 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -5,7 +5,7 @@ use std::{collections::HashMap, vec};
 use super::formats::lookup_format;
 use super::numeric::{rx_float_range, rx_int_range};
 use crate::{
-    api::{GrammarWithLexer, RegexSpec, TopLevelGrammar},
+    api::{GrammarWithLexer, RegexSpec, TopLevelGrammar, RegexId},
     GrammarBuilder, NodeRef,
 };
 
@@ -569,27 +569,44 @@ impl Compiler {
         additional_properties: &Value,
         required: Vec<String>,
     ) -> Result<NodeRef> {
-        let mut illegal_keys: Vec<String> = vec![];
+        let mut taken_names: Vec<String> = vec![];
         let mut items: Vec<(NodeRef, bool)> = vec![];
         for name in properties.keys().chain(required.iter().filter(|n| !properties.contains_key(n.as_str()))) {
             let property_schema = properties.get(name).unwrap_or(additional_properties);
             let is_required = required.contains(name);
+            // Quote (and escape) the name. TODO: probably overkill to use json_dumps here
+            let quoted_name = to_compact_json(&json!(name));
             if property_schema == &Value::Bool(false) {
                 if is_required {
                     bail!("Required property has 'false' schema: {}", name);
                 }
-                illegal_keys.push(name.clone());
+                // Even if the name won't be used for a property, it is still "taken" in that it can't be used for additionalProperties
+                taken_names.push(quoted_name);
                 continue;
             }
-            let name = self.builder.string(&format!("\"{}\"", name));
+            let name = self.builder.string(&quoted_name);
+            taken_names.push(quoted_name);
             let colon = self.builder.string(&self.options.key_separator);
             let the_rest = self.gen_json(property_schema)?;
             let item = self.builder.join(&[name, colon, the_rest]);
             items.push((item, is_required));
         }
         if additional_properties != &Value::Bool(false) {
-            // TODO: illegal keys and NOT(properties keys)
-            let name = self.json_simple_string();
+            let name = if taken_names.is_empty() {
+                self.json_simple_string()
+            } else {
+                let taken_name_ids =
+                    taken_names
+                    .iter()
+                    .map(|n| self.builder.regex.literal(n.to_string()))
+                    .collect::<Vec<RegexId>>();
+                let taken = self.builder.regex.select(taken_name_ids);
+                let not_taken = self.builder.regex.not(taken);
+                let valid = self.builder.regex.regex(r#""([^"\\]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})*""#.to_string());
+                let valid_and_not_taken = self.builder.regex.and(vec![valid, not_taken]);
+                let rx = RegexSpec::RegexId(valid_and_not_taken);
+                self.builder.lexeme(rx, false)
+            };
             let colon = self.builder.string(&self.options.key_separator);
             let the_rest = self.gen_json(additional_properties)?;
             let item = self.builder.join(&[name, colon, the_rest]);

From d576962d2b080c86e249403acbb3cb2ae5c38256 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Mon, 11 Nov 2024 15:50:03 -0800
Subject: [PATCH 21/73] type inference

---
 parser/src/json/compiler.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index f1c8f3f..d3cd79d 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -5,7 +5,7 @@ use std::{collections::HashMap, vec};
 use super::formats::lookup_format;
 use super::numeric::{rx_float_range, rx_int_range};
 use crate::{
-    api::{GrammarWithLexer, RegexSpec, TopLevelGrammar, RegexId},
+    api::{GrammarWithLexer, RegexSpec, TopLevelGrammar},
     GrammarBuilder, NodeRef,
 };
 
@@ -599,7 +599,7 @@ impl Compiler {
                     taken_names
                     .iter()
                     .map(|n| self.builder.regex.literal(n.to_string()))
-                    .collect::<Vec<RegexId>>();
+                    .collect::<Vec<_>>();
                 let taken = self.builder.regex.select(taken_name_ids);
                 let not_taken = self.builder.regex.not(taken);
                 let valid = self.builder.regex.regex(r#""([^"\\]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})*""#.to_string());

From 5b924aa67735b3c5ecebae80281f6a79a2ec1c2c Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Mon, 11 Nov 2024 15:53:50 -0800
Subject: [PATCH 22/73] json_dumps

---
 parser/src/json/compiler.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index d3cd79d..3e7dd35 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -20,7 +20,7 @@ pub struct JsonCompileOptions {
     pub whitespace_flexible: bool,
 }
 
-fn to_compact_json(target: &serde_json::Value) -> String {
+fn json_dumps(target: &serde_json::Value) -> String {
     serde_json::to_string(target).unwrap()
 }
 
@@ -385,7 +385,7 @@ impl Compiler {
             },
             _ => {
                 // let serde_json dump simple values
-                let const_str = to_compact_json(const_value);
+                let const_str = json_dumps(const_value);
                 Ok(self.builder.string(&const_str))
             }
         }
@@ -575,7 +575,7 @@ impl Compiler {
             let property_schema = properties.get(name).unwrap_or(additional_properties);
             let is_required = required.contains(name);
             // Quote (and escape) the name. TODO: probably overkill to use json_dumps here
-            let quoted_name = to_compact_json(&json!(name));
+            let quoted_name = json_dumps(&json!(name));
             if property_schema == &Value::Bool(false) {
                 if is_required {
                     bail!("Required property has 'false' schema: {}", name);

From e8eb74a52b136302b19ad8afe92b07d011fde423 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Mon, 11 Nov 2024 18:46:35 -0800
Subject: [PATCH 23/73] cache recursive calls of ordered_sequence

---
 parser/src/grammar_builder.rs |  2 +-
 parser/src/json/compiler.rs   | 29 +++++++++++++++++------------
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/parser/src/grammar_builder.rs b/parser/src/grammar_builder.rs
index 9ad835f..38f8113 100644
--- a/parser/src/grammar_builder.rs
+++ b/parser/src/grammar_builder.rs
@@ -6,7 +6,7 @@ use crate::api::{
     GrammarWithLexer, Node, NodeId, NodeProps, RegexId, RegexNode, RegexSpec, TopLevelGrammar,
 };
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
 pub struct NodeRef {
     idx: usize,
     grammar_id: u32,
diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index 3e7dd35..d74ae1b 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -614,13 +614,16 @@ impl Compiler {
             items.push((seq, false));
         }
         let opener = self.builder.string("{");
-        let inner = self.ordered_sequence(&items, false);
+        let inner = self.ordered_sequence(&items, false, &mut HashMap::<(&[(NodeRef, bool)], bool), NodeRef>::new());
         let closer = self.builder.string("}");
         Ok(self.builder.join(&[opener, inner, closer]))
     }
 
-
-    fn ordered_sequence(&mut self, items: &[(NodeRef, bool)], prefixed: bool) -> NodeRef {
+    fn ordered_sequence<'a>(&mut self, items: &'a[(NodeRef, bool)], prefixed: bool, cache: &mut HashMap::<(&'a[(NodeRef, bool)], bool), NodeRef>) -> NodeRef {
+        // Cache to reduce number of nodes from O(n^2) to O(n)
+        if let Some(node) = cache.get(&(items, prefixed)) {
+            return node.clone();
+        }
         if items.is_empty() {
             return self.builder.string("");
         }
@@ -628,10 +631,10 @@ impl Compiler {
         let (item, required) = items[0];
         let rest = &items[1..];
 
-        match (prefixed, required) {
+        let node = match (prefixed, required) {
             (true, true) => {
                 // If we know we have preceeding elements, we can safely just add a (',' + e)
-                let rest_seq = self.ordered_sequence(rest, true);
+                let rest_seq = self.ordered_sequence(rest, true, cache);
                 self.builder.join(&[comma, item, rest_seq])
             },
             (true, false) => {
@@ -639,27 +642,29 @@ impl Compiler {
                 // TODO optimization: if the rest is all optional, we can nest the rest in the optional
                 let comma_item = self.builder.join(&[comma, item]);
                 let optional_comma_item = self.builder.optional(comma_item);
-                let rest_seq = self.ordered_sequence(rest, true);
+                let rest_seq = self.ordered_sequence(rest, true, cache);
                 self.builder.join(&[optional_comma_item, rest_seq])
             },
             (false, true) => {
-                // No preceeding elements, so we just add the element (no comma)
-                let rest_seq = self.ordered_sequence(rest, true);
+                // No preceding elements, so we just add the element (no comma)
+                let rest_seq = self.ordered_sequence(rest, true, cache);
                 self.builder.join(&[item, rest_seq])
             },
             (false, false) => {
-                // No preceeding elements, but our element is optional. If we add the element, the remaining
+                // No preceding elements, but our element is optional. If we add the element, the remaining
                 // will be prefixed, else they are not.
                 // TODO: same nested optimization as above
-                let prefixed_rest = self.ordered_sequence(rest, true);
-                let unprefixed_rest = self.ordered_sequence(rest, false);
+                let prefixed_rest = self.ordered_sequence(rest, true, cache);
+                let unprefixed_rest = self.ordered_sequence(rest, false, cache);
                 let opts = [
                     self.builder.join(&[item, prefixed_rest]),
                     unprefixed_rest,
                 ];
                 self.builder.select(&opts)
             },
-        }
+        };
+        cache.insert((items, prefixed), node.clone());
+        node
     }
 
     fn sequence(&mut self, item: NodeRef) -> NodeRef {

From f648ebe9060f3485f7af70cc70d4892c490de5ad Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Mon, 11 Nov 2024 19:32:54 -0800
Subject: [PATCH 24/73] check number bounds

---
 parser/src/json/compiler.rs | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index d74ae1b..c5ed6a9 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -24,6 +24,17 @@ fn json_dumps(target: &serde_json::Value) -> String {
     serde_json::to_string(target).unwrap()
 }
 
+#[derive(Debug)]
+struct UnsatisfiableSchemaError {
+    message: String,
+}
+
+impl std::fmt::Display for UnsatisfiableSchemaError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Unsatisfiable schema: {}", self.message)
+    }
+}
+
 const TYPES: [&str; 7] = ["null", "boolean", "integer", "number", "string", "array", "object"];
 const KEYWORDS: [&str; 10] = [
     "anyOf",
@@ -97,6 +108,24 @@ fn validate_json_node_keys(node: &Value) -> Result<()> {
     Ok(())
 }
 
+fn check_number_bounds(minimum: Option<f64>, maximum: Option<f64>, exclusive_minimum: bool, exclusive_maximum: bool) -> Result<()> {
+    if let (Some(min), Some(max)) = (minimum, maximum) {
+        if min > max {
+            return Err(anyhow!(UnsatisfiableSchemaError {
+                message: format!("minimum ({}) is greater than maximum ({})", min, max),
+            }));
+        }
+        if min == max && (exclusive_minimum || exclusive_maximum) {
+            let minimum_repr = if exclusive_minimum { "exclusiveMinimum" } else { "minimum" };
+            let maximum_repr = if exclusive_maximum { "exclusiveMaximum" } else { "maximum" };
+            return Err(anyhow!(UnsatisfiableSchemaError {
+                message: format!("{} ({}) is equal to {} ({})", minimum_repr, min, maximum_repr, max),
+            }));
+        }
+    }
+    Ok(())
+}
+
 struct Compiler {
     builder: GrammarBuilder,
     options: JsonCompileOptions,
@@ -490,6 +519,7 @@ impl Compiler {
     }
 
     fn json_int(&mut self, minimum: Option<f64>, maximum: Option<f64>, exclusive_minimum: bool, exclusive_maximum: bool) -> Result<NodeRef> {
+        check_number_bounds(minimum, maximum, exclusive_minimum, exclusive_maximum)?;
         let minimum = match (minimum, exclusive_minimum) {
             (Some(min_val), true) => {
                 if min_val.fract() != 0.0 {
@@ -518,6 +548,7 @@ impl Compiler {
     }
 
     fn json_number(&mut self, minimum: Option<f64>, maximum: Option<f64>, exclusive_minimum: bool, exclusive_maximum: bool) -> Result<NodeRef> {
+        check_number_bounds(minimum, maximum, exclusive_minimum, exclusive_maximum)?;
         // TODO: handle errors in rx_float_range; currently it just panics
         let rx = rx_float_range(minimum, maximum, !exclusive_minimum, !exclusive_maximum);
         Ok(self.lexeme(&rx))

From b2fc186471a4f041964f0919bf4c9dfff566bda7 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Mon, 11 Nov 2024 21:55:16 -0800
Subject: [PATCH 25/73] a few UnsatisfiableSchemaError propagation cases

---
 parser/src/json/compiler.rs | 169 ++++++++++++++++++++++--------------
 1 file changed, 105 insertions(+), 64 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index c5ed6a9..11c8ad5 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -320,7 +320,9 @@ impl Compiler {
         }
 
         if json_schema.as_bool() == Some(false) {
-            bail!("'false' not supported as schema here");
+            return Err(anyhow!(UnsatisfiableSchemaError {
+                message: "false schema".to_string(),
+            }));
         }
 
         if let Some(json_schema) = json_schema.as_object() {
@@ -607,42 +609,60 @@ impl Compiler {
             let is_required = required.contains(name);
             // Quote (and escape) the name. TODO: probably overkill to use json_dumps here
             let quoted_name = json_dumps(&json!(name));
-            if property_schema == &Value::Bool(false) {
-                if is_required {
-                    bail!("Required property has 'false' schema: {}", name);
+            let property = match self.gen_json(property_schema) {
+                Ok(node) => node,
+                Err(e) => match e.downcast_ref::<UnsatisfiableSchemaError>() {
+                    // If it's not an UnsatisfiableSchemaError, just propagate it normally
+                    None => { return Err(e) },
+                    // Property is optional; don't raise UnsatisfiableSchemaError but mark name as taken
+                    Some(_) if !is_required => {
+                        taken_names.push(quoted_name);
+                        continue;
+                    },
+                    // Property is required; add context and propagate UnsatisfiableSchemaError
+                    Some(_) => {
+                        return Err(e.context(UnsatisfiableSchemaError {
+                            message: format!("required property '{}' is unsatisfiable", name),
+                        }));
+                    }
                 }
-                // Even if the name won't be used for a property, it is still "taken" in that it can't be used for additionalProperties
-                taken_names.push(quoted_name);
-                continue;
-            }
+            };
             let name = self.builder.string(&quoted_name);
             taken_names.push(quoted_name);
             let colon = self.builder.string(&self.options.key_separator);
-            let the_rest = self.gen_json(property_schema)?;
-            let item = self.builder.join(&[name, colon, the_rest]);
+            let item = self.builder.join(&[name, colon, property]);
             items.push((item, is_required));
         }
-        if additional_properties != &Value::Bool(false) {
-            let name = if taken_names.is_empty() {
-                self.json_simple_string()
-            } else {
-                let taken_name_ids =
-                    taken_names
-                    .iter()
-                    .map(|n| self.builder.regex.literal(n.to_string()))
-                    .collect::<Vec<_>>();
-                let taken = self.builder.regex.select(taken_name_ids);
-                let not_taken = self.builder.regex.not(taken);
-                let valid = self.builder.regex.regex(r#""([^"\\]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})*""#.to_string());
-                let valid_and_not_taken = self.builder.regex.and(vec![valid, not_taken]);
-                let rx = RegexSpec::RegexId(valid_and_not_taken);
-                self.builder.lexeme(rx, false)
-            };
-            let colon = self.builder.string(&self.options.key_separator);
-            let the_rest = self.gen_json(additional_properties)?;
-            let item = self.builder.join(&[name, colon, the_rest]);
-            let seq = self.sequence(item);
-            items.push((seq, false));
+
+        match self.gen_json(additional_properties) {
+            Err(e) => {
+                if e.downcast_ref::<UnsatisfiableSchemaError>().is_none() {
+                    // Propagate errors that aren't UnsatisfiableSchemaError
+                    return Err(e);
+                }
+                // Ignore UnsatisfiableSchemaError for additionalProperties
+            },
+            Ok(property) => {
+                let name = if taken_names.is_empty() {
+                    self.json_simple_string()
+                } else {
+                    let taken_name_ids =
+                        taken_names
+                        .iter()
+                        .map(|n| self.builder.regex.literal(n.to_string()))
+                        .collect::<Vec<_>>();
+                    let taken = self.builder.regex.select(taken_name_ids);
+                    let not_taken = self.builder.regex.not(taken);
+                    let valid = self.builder.regex.regex(r#""([^"\\]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})*""#.to_string());
+                    let valid_and_not_taken = self.builder.regex.and(vec![valid, not_taken]);
+                    let rx = RegexSpec::RegexId(valid_and_not_taken);
+                    self.builder.lexeme(rx, false)
+                };
+                let colon = self.builder.string(&self.options.key_separator);
+                let item = self.builder.join(&[name, colon, property]);
+                let seq = self.sequence(item);
+                items.push((seq, false));
+            }
         }
         let opener = self.builder.string("{");
         let inner = self.ordered_sequence(&items, false, &mut HashMap::<(&[(NodeRef, bool)], bool), NodeRef>::new());
@@ -712,6 +732,13 @@ impl Compiler {
         regex: Option<&str>,
         format: Option<&str>,
     ) -> Result<NodeRef> {
+        if let Some(max_length) = max_length {
+            if min_length > max_length {
+                return Err(anyhow!(UnsatisfiableSchemaError {
+                    message: format!("minLength ({}) is greater than maxLength ({})", min_length, max_length),
+                }));
+            }
+        }
 
         let mut regex = regex;
 
@@ -755,32 +782,34 @@ impl Compiler {
         min_items: u64,
         max_items: Option<u64>,
     ) -> Result<NodeRef> {
-        let anything_goes = json!({});
-        let item_schema = if item_schema.as_bool() == Some(true) {
-            &anything_goes
-        } else {
-            item_schema
-        };
-        let item_schema_is_false = item_schema.as_bool() == Some(false);
-
-        if item_schema_is_false && prefix_items.len() < min_items as usize {
-            bail!(
-                "PrefixItems has too few elements ({}) to satisfy minItems ({}) but no extra items were allowed",
-                prefix_items.len(),
-                min_items
-            );
-        }
-
-        if let Some(max_items_value) = max_items {
-            if max_items_value < min_items {
-                bail!(
-                    "maxItems ({}) can't be less than minItems ({})",
-                    max_items_value,
-                    min_items
-                );
+        let mut max_items = max_items;
+
+        if let Some(max_items) = max_items {
+            if min_items > max_items {
+                return Err(anyhow!(UnsatisfiableSchemaError {
+                    message: format!("minItems ({}) is greater than maxItems ({})", min_items, max_items),
+                }));
             }
         }
 
+        let additional_item_grm = match self.gen_json(item_schema) {
+            Ok(node) => Some(node),
+            Err(e) => match e.downcast_ref::<UnsatisfiableSchemaError>() {
+                // If it's not an UnsatisfiableSchemaError, just propagate it normally
+                None => { return Err(e) },
+                // Item is optional; don't raise UnsatisfiableSchemaError
+                Some(_) if prefix_items.len() >= min_items as usize => {
+                    None
+                },
+                // Item is required; add context and propagate UnsatisfiableSchemaError
+                Some(_) => {
+                    return Err(e.context(UnsatisfiableSchemaError {
+                        message: format!("required item is unsatisfiable"),
+                    }));
+                }
+            }
+        };
+
         let mut required_items = vec![];
         let mut optional_items = vec![];
 
@@ -794,16 +823,28 @@ impl Compiler {
                 required_items.push(self.gen_json(item)?);
             }
         } else {
-            let item_schema_compiled = if item_schema_is_false {
-                None
-            } else {
-                Some(self.gen_json(item_schema)?)
-            };
-
             for i in 0..n_to_add {
                 let item = if i < prefix_items.len() {
-                    self.gen_json(&prefix_items[i])?
-                } else if let Some(compiled) = &item_schema_compiled {
+                    match self.gen_json(&prefix_items[i]) {
+                        Ok(node) => node,
+                        Err(e) => match e.downcast_ref::<UnsatisfiableSchemaError>() {
+                            // If it's not an UnsatisfiableSchemaError, just propagate it normally
+                            None => { return Err(e) },
+                            // Item is optional; don't raise UnsatisfiableSchemaError.
+                            // Set max_items to the current index, as we can't satisfy any more items.
+                            Some(_) if i >= min_items as usize => {
+                                max_items = Some(i as u64);
+                                break;
+                            },
+                            // Item is required; add context and propagate UnsatisfiableSchemaError
+                            Some(_) => {
+                                return Err(e.context(UnsatisfiableSchemaError {
+                                    message: format!("prefixItems[{}] is unsatisfiable but minItems is {}", i, min_items),
+                                }));
+                            }
+                        }
+                    }
+                } else if let Some(compiled) = &additional_item_grm {
                     compiled.clone()
                 } else {
                     break;
@@ -816,9 +857,9 @@ impl Compiler {
                 }
             }
 
-            if max_items.is_none() && !item_schema_is_false {
+            if max_items.is_none() && !additional_item_grm.is_none() {
                 // Add an infinite tail of items
-                optional_items.push(self.sequence(item_schema_compiled.unwrap()));
+                optional_items.push(self.sequence(additional_item_grm.unwrap()));
             }
         }
 

From faa616250db6ee705cce6fd5c01ba621110e00b5 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 12 Nov 2024 09:04:41 -0800
Subject: [PATCH 26/73] ignore unsatisfiable schemas in anyOf

---
 parser/src/json/compiler.rs | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index 11c8ad5..11d0ba9 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -305,13 +305,35 @@ impl Compiler {
     }
 
     fn process_any_of(&mut self, obj: &Value) -> Result<NodeRef> {
-        let arr = obj
-            .as_array()
-            .ok_or_else(|| anyhow!("Expected array in anyOf, got: {}", limited_str(obj)))?
-            .iter()
+        let arr = obj.as_array().ok_or_else(|| anyhow!("Expected array in anyOf, got: {}", limited_str(obj)))?;
+        if arr.is_empty() {
+            return Err(anyhow!(UnsatisfiableSchemaError {
+                message: "anyOf has no schemas".to_string(),
+            }));
+        }
+        // Unwrap, filtering out unsatisfiable schemas
+        let options = arr.iter()
             .map(|json_schema| self.gen_json(json_schema))
-            .collect::<Result<Vec<_>>>()?;
-        Ok(self.builder.select(&arr))
+            .filter_map(|res| {
+                match res {
+                    Ok(node) => Some(Ok(node)),
+                    Err(e) => match e.downcast_ref::<UnsatisfiableSchemaError>() {
+                        // If it's not an UnsatisfiableSchemaError, just propagate it normally
+                        None => { Some(Err(e)) },
+                        // Ignore unsatisfiable schemas
+                        Some(_) => None,
+                    }
+                }
+            })
+            .collect::<Result<Vec<NodeRef>>>()?;
+        if options.is_empty() {
+            // If all schemas are unsatisfiable, return an error
+            // We can't really point to any specific schema at fault, so just dump the whole anyOf
+            return Err(anyhow!(UnsatisfiableSchemaError {
+                message: format!("all schemas in anyOf are unsatisfiable: {}", json_dumps(obj)),
+            }));
+        }
+        Ok(self.builder.select(&options))
     }
 
     fn gen_json(&mut self, json_schema: &Value) -> Result<NodeRef> {

From aeff51251be447ba8489cd3cb3fe669fa32a0174 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 12 Nov 2024 09:37:05 -0800
Subject: [PATCH 27/73] remove substring.rs (for separate PR)

---
 parser/src/json/mod.rs       |   1 -
 parser/src/json/substring.rs | 150 -----------------------------------
 2 files changed, 151 deletions(-)
 delete mode 100644 parser/src/json/substring.rs

diff --git a/parser/src/json/mod.rs b/parser/src/json/mod.rs
index 832988d..f40ea97 100644
--- a/parser/src/json/mod.rs
+++ b/parser/src/json/mod.rs
@@ -1,4 +1,3 @@
 pub mod compiler;
 mod formats;
 mod numeric;
-mod substring;
diff --git a/parser/src/json/substring.rs b/parser/src/json/substring.rs
deleted file mode 100644
index a1f29a6..0000000
--- a/parser/src/json/substring.rs
+++ /dev/null
@@ -1,150 +0,0 @@
-use std::{collections::HashMap, vec};
-use derivre::{ExprRef, Regex, RegexAst, RegexBuilder};
-
-#[derive(Debug)]
-struct State {
-    len: usize,
-    link: Option<usize>,
-    next: HashMap<char, usize>,
-}
-
-/// For details see https://en.wikipedia.org/wiki/Suffix_automaton.
-/// Implementation is based on https://cp-algorithms.com/string/suffix-automaton.html
-struct SuffixAutomaton {
-    states: Vec<State>,
-    last: usize,
-}
-
-impl SuffixAutomaton {
-    fn new() -> Self {
-        let init_state = State {
-            len: 0,
-            link: None,
-            next: HashMap::new(),
-        };
-        SuffixAutomaton {
-            states: vec![init_state],
-            last: 0,
-        }
-    }
-
-    fn from_string(s: &str) -> Self {
-        let mut sa = SuffixAutomaton::new();
-        for c in s.chars() {
-            sa.extend(c);
-        }
-        sa
-    }
-
-    fn extend(&mut self, c: char) {
-        let cur_index = self.states.len();
-        self.states.push(
-            State {
-                len: self.states[self.last].len + 1,
-                link: None,
-                next: HashMap::new(),
-            }
-        );
-
-        let mut p = Some(self.last);
-        while let Some(pp) = p{
-            if self.states[pp].next.contains_key(&c) {
-                break;
-            }
-            self.states[pp].next.insert(c, cur_index);
-            p = self.states[pp].link;
-        }
-
-        if let Some(pp) = p {
-            let q = self.states[pp].next[&c];
-            if self.states[pp].len + 1 == self.states[q].len {
-                self.states[cur_index].link = Some(q);
-            } else {
-                let clone_index = self.states.len();
-                self.states.push(
-                    State {
-                        len: self.states[pp].len + 1,
-                        link: self.states[q].link,
-                        next: self.states[q].next.clone(),
-                    }
-                );
-                while let Some(ppp) = p {
-                    if self.states[ppp].next[&c] == q {
-                        self.states[ppp].next.insert(c, clone_index);
-                    } else {
-                        break;
-                    }
-                    p = self.states[ppp].link;
-                }
-                self.states[q].link = Some(clone_index);
-                self.states[cur_index].link = Some(clone_index);
-            }
-        } else {
-            self.states[cur_index].link = Some(0);
-        }
-        self.last = cur_index;
-    }
-}
-
-pub fn substring(string: &str) -> Result<Regex, anyhow::Error> {
-    let sa = SuffixAutomaton::from_string(string);
-    let mut builder = RegexBuilder::new();
-    let mut state_stack = vec![0];
-    let mut node_cache: HashMap<usize, ExprRef> = HashMap::new();
-
-    while let Some(state_index) = state_stack.last() {
-        let state = &sa.states[*state_index];
-        if node_cache.contains_key(state_index) {
-            state_stack.pop();
-            continue;
-        }
-
-        if state.next.is_empty() {
-            let expr = builder.mk(&RegexAst::EmptyString)?;
-            node_cache.insert(*state_index, expr);
-            state_stack.pop();
-            continue;
-        }
-
-        let unprocessed_children = state.next
-            .values()
-            .filter(|child_index| !node_cache.contains_key(child_index))
-            .collect::<Vec<_>>();
-        if !unprocessed_children.is_empty() {
-            state_stack.extend(unprocessed_children);
-            continue;
-        }
-
-        let mut options = state.next
-            .keys()
-            .map(|c| {
-                RegexAst::Concat(vec![
-                    RegexAst::Literal(c.to_string()),
-                    RegexAst::ExprRef(node_cache[&state.next[c]]),
-                ])
-            })
-            .collect::<Vec<_>>();
-        options.push(RegexAst::EmptyString);
-        let node = RegexAst::Or(options);
-        let expr = builder.mk(&node)?;
-        node_cache.insert(*state_index, expr);
-        state_stack.pop();
-    }
-    Ok(builder.to_regex(node_cache[&0]))
-}
-
-mod tests {
-    use super::substring;
-
-    #[test]
-    fn test_substring() {
-        let regex = substring("abacaba").unwrap();
-        assert_eq!(regex.clone().is_match(""), true);
-        assert_eq!(regex.clone().is_match("acab"), true);
-        assert_eq!(regex.clone().is_match("abaca"), true);
-        assert_eq!(regex.clone().is_match("acaba"), true);
-        assert_eq!(regex.clone().is_match("abacaba"), true);
-        assert_eq!(regex.clone().is_match("acabab"), false);
-        assert_eq!(regex.clone().is_match("cabaca"), false);
-    }
-}

From a3b091240af833fd26108c4b21da5798bc086004 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 12 Nov 2024 12:57:12 -0800
Subject: [PATCH 28/73] type inference

---
 parser/src/json/compiler.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index 11d0ba9..a1056e9 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -687,7 +687,7 @@ impl Compiler {
             }
         }
         let opener = self.builder.string("{");
-        let inner = self.ordered_sequence(&items, false, &mut HashMap::<(&[(NodeRef, bool)], bool), NodeRef>::new());
+        let inner = self.ordered_sequence(&items, false, &mut HashMap::new());
         let closer = self.builder.string("}");
         Ok(self.builder.join(&[opener, inner, closer]))
     }

From ce89057f7f844d51f5cd55c5dc067be502dd7d6a Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 12 Nov 2024 12:57:52 -0800
Subject: [PATCH 29/73] drop todo

---
 parser/src/json/compiler.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index a1056e9..2bfdf2e 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -629,7 +629,7 @@ impl Compiler {
         for name in properties.keys().chain(required.iter().filter(|n| !properties.contains_key(n.as_str()))) {
             let property_schema = properties.get(name).unwrap_or(additional_properties);
             let is_required = required.contains(name);
-            // Quote (and escape) the name. TODO: probably overkill to use json_dumps here
+            // Quote (and escape) the name
             let quoted_name = json_dumps(&json!(name));
             let property = match self.gen_json(property_schema) {
                 Ok(node) => node,

From b94a085c1a4a37ee6c273c6bb49dd22a91b06018 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Wed, 13 Nov 2024 16:28:18 -0800
Subject: [PATCH 30/73] normalize

---
 parser/Cargo.lock            |   1 +
 parser/Cargo.toml            |   1 +
 parser/src/json/mod.rs       |   1 +
 parser/src/json/normalize.rs | 642 +++++++++++++++++++++++++++++++++++
 rust/Cargo.lock              |   1 +
 sample_parser/Cargo.lock     |   1 +
 6 files changed, 647 insertions(+)
 create mode 100644 parser/src/json/normalize.rs

diff --git a/parser/Cargo.lock b/parser/Cargo.lock
index 79935d9..adfc17c 100644
--- a/parser/Cargo.lock
+++ b/parser/Cargo.lock
@@ -562,6 +562,7 @@ dependencies = [
  "anyhow",
  "cbindgen",
  "derivre",
+ "indexmap",
  "instant",
  "jsonschema",
  "lazy_static",
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index 1d7a763..f37f3bf 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -15,6 +15,7 @@ jsonschema = { version = "0.24.0", default-features = false, optional = true }
 url = "2.5.2"
 lazy_static = { version = "1.5.0", optional = true }
 regex-syntax = "0.8.5"
+indexmap = "2.6.0"
 
 [features]
 default = ["jsonschema_validation", "lark"]
diff --git a/parser/src/json/mod.rs b/parser/src/json/mod.rs
index f40ea97..0509f4e 100644
--- a/parser/src/json/mod.rs
+++ b/parser/src/json/mod.rs
@@ -1,3 +1,4 @@
 pub mod compiler;
 mod formats;
+mod normalize;
 mod numeric;
diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
new file mode 100644
index 0000000..d56045c
--- /dev/null
+++ b/parser/src/json/normalize.rs
@@ -0,0 +1,642 @@
+use anyhow::{anyhow, bail, Result};
+use indexmap::IndexMap;
+use serde_json::{Map, Value};
+
+const TYPES: [&str; 7] = [
+    "null", "boolean", "integer", "number", "string", "array", "object",
+];
+
+#[derive(Debug, PartialEq, Clone)]
+enum Schema {
+    Any,
+    Unsatisfiable {
+        reason: String,
+    },
+    Null,
+    Boolean,
+    Number {
+        minimum: Option<f64>,
+        maximum: Option<f64>,
+        exclusive_minimum: Option<f64>,
+        exclusive_maximum: Option<f64>,
+        integer: bool,
+    },
+    String {
+        min_length: u64,
+        max_length: Option<u64>,
+        pattern: Option<String>,
+        format: Option<String>,
+    },
+    Array {
+        min_items: u64,
+        max_items: Option<u64>,
+        prefix_items: Vec<Schema>,
+        items: Option<Box<Schema>>,
+    },
+    Object {
+        properties: IndexMap<String, Schema>,
+        additional_properties: Option<Box<Schema>>,
+        required: Vec<String>,
+    },
+    Const {
+        value: Value,
+    },
+    Enum {
+        options: Vec<Value>,
+    },
+    AnyOf {
+        options: Vec<Schema>,
+    },
+    OneOf {
+        options: Vec<Schema>,
+    },
+    Ref(Box<Schema>),
+}
+
+fn normalize(schema: &Value) -> Result<Schema> {
+    match schema.as_bool() {
+        Some(true) => return Ok(Schema::Any),
+        Some(false) => {
+            return Ok(Schema::Unsatisfiable {
+                reason: "schema is false".to_string(),
+            })
+        }
+        None => {}
+    }
+
+    // Get the schema as an object
+    // TODO: validate against metaschema & check for unimplemented keys
+    let schema = schema
+        .as_object()
+        .ok_or_else(|| anyhow!("schema must be an object or boolean"))?;
+
+    if schema.is_empty() {
+        // TODO: should be ok to have ignored keys here
+        return Ok(Schema::Any);
+    }
+
+    if let Some(instance) = schema.get("const") {
+        // TODO: validate the instance against the schema, maybe returning Schema::Unsatisfiable
+        return Ok(Schema::Const {
+            value: instance.clone(),
+        });
+    }
+
+    if let Some(instances) = schema.get("enum") {
+        let instances = instances
+            .as_array()
+            .ok_or_else(|| anyhow!("enum must be an array"))?;
+        if instances.is_empty() {
+            return Ok(Schema::Unsatisfiable {
+                reason: "enum is empty".to_string(),
+            });
+        }
+        // TODO: validate the instances against the schema, maybe returning Schema::Unsatisfiable
+        return Ok(Schema::Enum {
+            options: instances.clone(),
+        });
+    }
+
+    // Make a mutable copy of the schema so we can modify it
+    let mut schema = schema.clone();
+
+    if let Some(all_of) = schema.remove("allOf") {
+        let all_of = all_of
+            .as_array()
+            .ok_or_else(|| anyhow!("allOf must be an array"))?;
+        let siblings = normalize(&Value::Object(schema))?;
+        if all_of.is_empty() || matches!(siblings, Schema::Unsatisfiable { .. }) {
+            return Ok(siblings);
+        }
+        let options = all_of
+            .iter()
+            .map(normalize)
+            .collect::<Result<Vec<Schema>>>()?;
+        let merged = merge(options.iter().chain(vec![&siblings]).collect())?;
+        return Ok(merged);
+    }
+
+    if let Some(any_of) = schema.remove("anyOf") {
+        let any_of = any_of
+            .as_array()
+            .ok_or_else(|| anyhow!("anyOf must be an array"))?;
+        let siblings = normalize(&Value::Object(schema))?;
+        if matches!(siblings, Schema::Unsatisfiable { .. }) {
+            return Ok(siblings);
+        }
+        if any_of.is_empty() {
+            return Ok(Schema::Unsatisfiable {
+                reason: "anyOf is empty".to_string(),
+            });
+        }
+        let options = any_of
+            .iter()
+            .filter_map(|value| match normalize(value) {
+                Ok(subschema) => {
+                    let merged = merge(vec![&subschema, &siblings]);
+                    match merged {
+                        Ok(Schema::Unsatisfiable { .. }) => None,
+                        _ => Some(merged),
+                    }
+                }
+                Err(_) => None,
+            })
+            .collect::<Result<Vec<Schema>>>()?;
+        if options.is_empty() {
+            return Ok(Schema::Unsatisfiable {
+                reason: "all anyOf options are unsatisfiable".to_string(),
+            });
+        }
+        return Ok(Schema::AnyOf { options });
+    }
+
+    // TODO: refactor to share code with anyOf
+    if let Some(one_of) = schema.remove("oneOf") {
+        let one_of = one_of
+            .as_array()
+            .ok_or_else(|| anyhow!("oneOf must be an array"))?;
+        let siblings = normalize(&Value::Object(schema))?;
+        if matches!(siblings, Schema::Unsatisfiable { .. }) {
+            return Ok(siblings);
+        }
+        if one_of.is_empty() {
+            return Ok(Schema::Unsatisfiable {
+                reason: "oneOf is empty".to_string(),
+            });
+        }
+        let options = one_of
+            .iter()
+            .filter_map(|value| match normalize(value) {
+                Ok(subschema) => {
+                    let merged = merge(vec![&subschema, &siblings]);
+                    match merged {
+                        Ok(Schema::Unsatisfiable { .. }) => None,
+                        _ => Some(merged),
+                    }
+                }
+                Err(_) => None,
+            })
+            .collect::<Result<Vec<Schema>>>()?;
+        if options.is_empty() {
+            return Ok(Schema::Unsatisfiable {
+                reason: "all oneOf options are unsatisfiable".to_string(),
+            });
+        }
+        return Ok(Schema::OneOf { options });
+    }
+
+    if let Some(_) = schema.remove("$ref") {
+        bail!("Ref not implemented")
+    }
+
+    let types = match schema.remove("type") {
+        Some(Value::String(tp)) => {
+            return normalize_type(&schema, &tp);
+        }
+        Some(Value::Array(types)) => types
+            .iter()
+            .map(|tp| match tp.as_str() {
+                Some(tp) => Ok(tp.to_string()),
+                None => bail!("type must be a string"),
+            })
+            .collect::<Result<Vec<String>>>()?,
+        Some(_) => bail!("type must be a string or array"),
+        None => TYPES.iter().map(|s| s.to_string()).collect(),
+    };
+    Ok(Schema::AnyOf {
+        options: types
+            .iter()
+            .map(|tp| normalize_type(&schema, &tp))
+            .collect::<Result<Vec<Schema>>>()?,
+    })
+}
+
+fn normalize_type(schema: &Map<String, Value>, tp: &str) -> Result<Schema> {
+    match tp {
+        "null" => Ok(Schema::Null),
+        "boolean" => Ok(Schema::Boolean),
+        "number" | "integer" => {
+            let minimum = schema.get("minimum").and_then(|v| v.as_f64());
+            let maximum = schema.get("maximum").and_then(|v| v.as_f64());
+            let exclusive_minimum = schema.get("exclusiveMinimum").and_then(|v| v.as_f64());
+            let exclusive_maximum = schema.get("exclusiveMaximum").and_then(|v| v.as_f64());
+            Ok(Schema::Number {
+                minimum,
+                maximum,
+                exclusive_minimum,
+                exclusive_maximum,
+                integer: tp == "integer",
+            })
+        }
+        "string" => {
+            let min_length = schema
+                .get("minLength")
+                .and_then(|v| v.as_u64())
+                .unwrap_or(0);
+            let max_length = schema.get("maxLength").and_then(|v| v.as_u64());
+            let pattern = schema
+                .get("pattern")
+                .and_then(|v| v.as_str())
+                .map(|s| s.to_string());
+            let format = schema
+                .get("format")
+                .and_then(|v| v.as_str())
+                .map(|s| s.to_string());
+            Ok(Schema::String {
+                min_length,
+                max_length,
+                pattern,
+                format,
+            })
+        }
+        "array" => {
+            let min_items = schema.get("minItems").and_then(|v| v.as_u64()).unwrap_or(0);
+            let max_items = schema.get("maxItems").and_then(|v| v.as_u64());
+            let prefix_items = schema
+                .get("prefixItems")
+                .map(|v| {
+                    v.as_array()
+                        .ok_or_else(|| anyhow!("prefixItems must be an array"))
+                })
+                .transpose()?
+                .map(|items| {
+                    items
+                        .iter()
+                        .map(|item| normalize(item))
+                        .collect::<Result<Vec<Schema>>>()
+                })
+                .transpose()?
+                .unwrap_or_default();
+            let items = schema.get("items").map(|v| normalize(v)).transpose()?;
+            Ok(Schema::Array {
+                min_items,
+                max_items,
+                prefix_items,
+                items: items.map(Box::new),
+            })
+        }
+        "object" => {
+            let properties = schema
+                .get("properties")
+                .map(|v| {
+                    v.as_object()
+                        .ok_or_else(|| anyhow!("properties must be an object"))
+                })
+                .transpose()?
+                .map(|props| {
+                    props
+                        .iter()
+                        .map(|(k, v)| normalize(v).map(|schema| (k.clone(), schema)))
+                        .collect::<Result<IndexMap<String, Schema>>>()
+                })
+                .transpose()?
+                .unwrap_or_default();
+            let additional_properties = schema
+                .get("additionalProperties")
+                .map(|v| normalize(v))
+                .transpose()?;
+            let required = schema
+                .get("required")
+                .map(|v| {
+                    v.as_array()
+                        .ok_or_else(|| anyhow!("required must be an array"))
+                })
+                .transpose()?
+                .map(|items| {
+                    items
+                        .iter()
+                        .map(|item| {
+                            item.as_str()
+                                .ok_or_else(|| anyhow!("required items must be strings"))
+                                .map(|s| s.to_string())
+                        })
+                        .collect::<Result<Vec<String>>>()
+                })
+                .transpose()?
+                .unwrap_or_default();
+            Ok(Schema::Object {
+                properties,
+                additional_properties: additional_properties.map(Box::new),
+                required,
+            })
+        }
+        _ => bail!("Invalid type: {}", tp),
+    }
+}
+
+fn merge(schemas: Vec<&Schema>) -> Result<Schema> {
+    if schemas.is_empty() {
+        bail!("merge called with empty list")
+    }
+    if schemas.iter().all(|schema| matches!(schema, Schema::Any)) {
+        return Ok(Schema::Any);
+    }
+    if schemas
+        .iter()
+        .any(|schema| matches!(schema, Schema::Unsatisfiable { .. }))
+    {
+        // Return the first unsatisfiable schema for debug-ability
+        // TODO: think through ownership and don't return a clone if possible
+        return Ok(schemas[0].to_owned());
+    }
+    // TODO: can we avoid cloning here?
+    let mut merged = schemas[0].to_owned();
+    for subschema in &schemas[1..] {
+        merged = merge_two((&merged, subschema))?;
+    }
+    Ok(merged.to_owned())
+}
+
+fn merge_two(schemas: (&Schema, &Schema)) -> Result<Schema> {
+    match schemas {
+        (Schema::Any, _) => Ok(schemas.1.to_owned()),
+        (_, Schema::Any) => Ok(schemas.0.to_owned()),
+        (Schema::Unsatisfiable { .. }, _) => Ok(schemas.0.to_owned()),
+        (_, Schema::Unsatisfiable { .. }) => Ok(schemas.1.to_owned()),
+        (
+            Schema::OneOf { .. } | Schema::AnyOf { .. },
+            Schema::OneOf { .. } | Schema::AnyOf { .. },
+        ) => merge_oneof_anyof(schemas),
+        (Schema::OneOf { options }, _) => {
+            let new_options = options
+                .iter()
+                .map(|option| merge_two((option, schemas.1)))
+                .collect::<Result<Vec<Schema>>>()?;
+            Ok(Schema::OneOf {
+                options: new_options,
+            })
+        }
+        (_, Schema::OneOf { options }) => {
+            let new_options = options
+                .iter()
+                .map(|option| merge_two((schemas.0, option)))
+                .collect::<Result<Vec<Schema>>>()?;
+            Ok(Schema::OneOf {
+                options: new_options,
+            })
+        }
+        (Schema::AnyOf { options }, _) => {
+            let new_options = options
+                .iter()
+                .map(|option| merge_two((option, schemas.1)))
+                .collect::<Result<Vec<Schema>>>()?;
+            Ok(Schema::AnyOf {
+                options: new_options,
+            })
+        }
+        (_, Schema::AnyOf { options }) => {
+            let new_options = options
+                .iter()
+                .map(|option| merge_two((schemas.0, option)))
+                .collect::<Result<Vec<Schema>>>()?;
+            Ok(Schema::AnyOf {
+                options: new_options,
+            })
+        }
+        (Schema::Null, Schema::Null) => Ok(Schema::Null),
+        (Schema::Boolean, Schema::Boolean) => Ok(Schema::Boolean),
+        (
+            Schema::Number {
+                minimum: min1,
+                maximum: max1,
+                exclusive_minimum: emin1,
+                exclusive_maximum: emax1,
+                integer: int1,
+            },
+            Schema::Number {
+                minimum: min2,
+                maximum: max2,
+                exclusive_minimum: emin2,
+                exclusive_maximum: emax2,
+                integer: int2,
+            },
+        ) => Ok(Schema::Number {
+            minimum: opt_max(*min1, *min2),
+            maximum: opt_min(*max1, *max2),
+            exclusive_minimum: opt_max(*emin1, *emin2),
+            exclusive_maximum: opt_min(*emax1, *emax2),
+            integer: *int1 && *int2,
+        }),
+        (
+            Schema::String {
+                min_length: min1,
+                max_length: max1,
+                pattern: pattern1,
+                format: format1,
+            },
+            Schema::String {
+                min_length: min2,
+                max_length: max2,
+                pattern: pattern2,
+                format: format2,
+            },
+        ) => Ok(Schema::String {
+            min_length: *min1.max(min2),
+            max_length: opt_min(*max1, *max2),
+            pattern: match (pattern1, pattern2) {
+                (None, None) => None,
+                (None, Some(r)) => Some(r.clone()),
+                (Some(r), None) => Some(r.clone()),
+                (Some(r1), Some(r2)) => {
+                    if r1 == r2 {
+                        Some(r1.clone())
+                    } else {
+                        bail!("intersection of patterns not implemented")
+                    }
+                }
+            },
+            format: match (format1, format2) {
+                (None, None) => None,
+                (None, Some(fmt)) => Some(fmt.clone()),
+                (Some(fmt), None) => Some(fmt.clone()),
+                (Some(fmt1), Some(fmt2)) => {
+                    if fmt1 == fmt2 {
+                        Some(fmt1.clone())
+                    } else {
+                        bail!("intersection of formats not implemented")
+                    }
+                }
+            },
+        }),
+        (
+            Schema::Array {
+                min_items: min1,
+                max_items: max1,
+                prefix_items: prefix1,
+                items: items1,
+            },
+            Schema::Array {
+                min_items: min2,
+                max_items: max2,
+                prefix_items: prefix2,
+                items: items2,
+            },
+        ) => Ok(Schema::Array {
+            min_items: *min1.max(min2),
+            max_items: opt_min(*max1, *max2),
+            prefix_items: zip_default(prefix1, prefix2, &Schema::Any)
+                .iter()
+                .map(|(item1, item2)| merge_two((item1, item2)))
+                .collect::<Result<Vec<Schema>>>()?,
+            items: match (items1, items2) {
+                (None, None) => None,
+                (None, Some(item)) => Some(Box::new(*item.clone())),
+                (Some(item), None) => Some(Box::new(*item.clone())),
+                (Some(item1), Some(item2)) => Some(Box::new(merge_two((&item1, &item2))?)),
+            },
+        }),
+        (
+            Schema::Object {
+                properties: props1,
+                additional_properties: add1,
+                required: req1,
+            },
+            Schema::Object {
+                properties: props2,
+                additional_properties: add2,
+                required: req2,
+            },
+        ) => {
+            let mut new_props = IndexMap::new();
+            for (key) in props1.keys().chain(props2.keys()) {
+                let new_schema = match (props1.get(key), props2.get(key), add1, add2) {
+                    (Some(schema1), Some(schema2), _, _) => merge_two((schema1, schema2))?,
+                    (Some(schema1), None, _, Some(add)) => merge_two((schema1, &add))?,
+                    (None, Some(schema2), Some(add), _) => merge_two((&add, schema2))?,
+                    (Some(schema1), None, _, None) => schema1.to_owned(),
+                    (None, Some(schema2), None, _) => schema2.to_owned(),
+                    (None, None, _, _) => bail!("should not happen"),
+                };
+                new_props.insert(key.clone(), new_schema);
+            }
+            Ok(Schema::Object {
+                properties: new_props,
+                additional_properties: match (add1, add2) {
+                    (None, None) => None,
+                    (None, Some(add)) => Some(Box::new(*add.clone())),
+                    (Some(add), None) => Some(Box::new(*add.clone())),
+                    (Some(add1), Some(add2)) => Some(Box::new(merge_two((&add1, &add2))?)),
+                },
+                required: req1.iter().chain(req2.iter()).cloned().collect(),
+            })
+        }
+        //TODO: get types for error message
+        _ => Ok(Schema::Unsatisfiable {
+            reason: "incompatible types".to_string(),
+        }),
+    }
+}
+
+fn zip_default<'a, T>(arr1: &'a [T], arr2: &'a [T], default: &'a T) -> Vec<(&'a T, &'a T)> {
+    let iter1 = arr1.iter().chain(std::iter::repeat(default));
+    let iter2 = arr2.iter().chain(std::iter::repeat(default));
+    iter1.zip(iter2).take(arr1.len().max(arr2.len())).collect()
+}
+
+fn opt_max<T: PartialOrd>(a: Option<T>, b: Option<T>) -> Option<T> {
+    match (a, b) {
+        (Some(a), Some(b)) => {
+            if a >= b {
+                Some(a)
+            } else {
+                Some(b)
+            }
+        }
+        (Some(a), None) => Some(a),
+        (None, Some(b)) => Some(b),
+        (None, None) => None,
+    }
+}
+
+fn opt_min<T: PartialOrd>(a: Option<T>, b: Option<T>) -> Option<T> {
+    match (a, b) {
+        (Some(a), Some(b)) => {
+            if a <= b {
+                Some(a)
+            } else {
+                Some(b)
+            }
+        }
+        (Some(a), None) => Some(a),
+        (None, Some(b)) => Some(b),
+        (None, None) => None,
+    }
+}
+
+fn merge_oneof_anyof(schemas: (&Schema, &Schema)) -> Result<Schema> {
+    let (schema1_options, schema2_options) = match schemas {
+        (
+            Schema::AnyOf { options: options1 } | Schema::OneOf { options: options1 },
+            Schema::AnyOf { options: options2 } | Schema::OneOf { options: options2 },
+        ) => (options1, options2),
+        _ => bail!("merge_oneof_anyof called with invalid schemas"),
+    };
+    let mut new_options = Vec::new();
+    for item1 in schema1_options {
+        for item2 in schema2_options {
+            let merged_item = merge(vec![item1, item2])?;
+            new_options.push(merged_item);
+        }
+    }
+    let valid_options = new_options
+        .iter()
+        .filter_map(|schema| match schema {
+            Schema::Unsatisfiable { .. } => None,
+            _ => Some(schema.to_owned()),
+        })
+        .collect::<Vec<Schema>>();
+    if valid_options.is_empty() {
+        // Return the first unsatisfiable schema for debug-ability
+        // TODO: think through ownership and don't return a clone if possible
+        return Ok(new_options[0].clone());
+    }
+
+    // OneOf takes precedence over AnyOf
+    if matches!(schemas.0, Schema::OneOf { .. }) || matches!(schemas.1, Schema::OneOf { .. }) {
+        return Ok(Schema::OneOf {
+            options: valid_options,
+        });
+    }
+    Ok(Schema::AnyOf {
+        options: valid_options,
+    })
+}
+
+mod test {
+    use super::*;
+    use regex_syntax::ast::print;
+    use serde_json::json;
+
+    #[test]
+    fn test_normalize() {
+        let schema = json!({
+            "minLength": 1,
+            "maxLength": 10,
+            "pattern": "^[a-z]+$",
+            "format": "email",
+            "minimum": 10,
+        });
+        let schema = normalize(&schema).unwrap();
+        println!("{:?}", schema);
+    }
+
+    #[test]
+    fn test_normalize_2() {
+        let schema = json!({
+            "anyOf": [
+                {
+                    "type": "string",
+                    "minLength": 1,
+                },
+                {
+                    "type": "number",
+                    "minimum": 10,
+                },
+            ],
+            "pattern": "^[a-z]+$",
+            "maximum": 11
+        });
+        let schema = normalize(&schema).unwrap();
+        println!("{:?}", schema);
+    }
+}
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 9284572..11cc46c 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -579,6 +579,7 @@ dependencies = [
  "anyhow",
  "cbindgen",
  "derivre",
+ "indexmap",
  "instant",
  "jsonschema",
  "lazy_static",
diff --git a/sample_parser/Cargo.lock b/sample_parser/Cargo.lock
index 1c6a8f2..2d41c27 100644
--- a/sample_parser/Cargo.lock
+++ b/sample_parser/Cargo.lock
@@ -861,6 +861,7 @@ dependencies = [
  "anyhow",
  "cbindgen",
  "derivre",
+ "indexmap",
  "instant",
  "jsonschema",
  "lazy_static",

From ef6c0668fc1ff3b4bf7b8a58e486b4c08c79dd25 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Wed, 13 Nov 2024 18:20:26 -0800
Subject: [PATCH 31/73] filter

---
 parser/src/json/normalize.rs | 68 +++++++++++++-----------------------
 1 file changed, 24 insertions(+), 44 deletions(-)

diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index d56045c..8020f1d 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -203,12 +203,20 @@ fn normalize(schema: &Value) -> Result<Schema> {
         Some(_) => bail!("type must be a string or array"),
         None => TYPES.iter().map(|s| s.to_string()).collect(),
     };
-    Ok(Schema::AnyOf {
-        options: types
-            .iter()
-            .map(|tp| normalize_type(&schema, &tp))
-            .collect::<Result<Vec<Schema>>>()?,
-    })
+    let options = types
+        .iter()
+        .map(|tp| normalize_type(&schema, &tp))
+        .filter_map(|result| match result {
+            Ok(Schema::Unsatisfiable { .. }) => None,
+            _ => Some(result),
+        })
+        .collect::<Result<Vec<Schema>>>()?;
+    if options.is_empty() {
+        return Ok(Schema::Unsatisfiable {
+            reason: "no valid types".to_string(),
+        });
+    }
+    Ok(Schema::AnyOf { options })
 }
 
 fn normalize_type(schema: &Map<String, Value>, tp: &str) -> Result<Schema> {
@@ -357,42 +365,12 @@ fn merge_two(schemas: (&Schema, &Schema)) -> Result<Schema> {
             Schema::OneOf { .. } | Schema::AnyOf { .. },
             Schema::OneOf { .. } | Schema::AnyOf { .. },
         ) => merge_oneof_anyof(schemas),
-        (Schema::OneOf { options }, _) => {
-            let new_options = options
-                .iter()
-                .map(|option| merge_two((option, schemas.1)))
-                .collect::<Result<Vec<Schema>>>()?;
-            Ok(Schema::OneOf {
-                options: new_options,
-            })
-        }
-        (_, Schema::OneOf { options }) => {
-            let new_options = options
-                .iter()
-                .map(|option| merge_two((schemas.0, option)))
-                .collect::<Result<Vec<Schema>>>()?;
-            Ok(Schema::OneOf {
-                options: new_options,
-            })
-        }
-        (Schema::AnyOf { options }, _) => {
-            let new_options = options
-                .iter()
-                .map(|option| merge_two((option, schemas.1)))
-                .collect::<Result<Vec<Schema>>>()?;
-            Ok(Schema::AnyOf {
-                options: new_options,
-            })
-        }
-        (_, Schema::AnyOf { options }) => {
-            let new_options = options
-                .iter()
-                .map(|option| merge_two((schemas.0, option)))
-                .collect::<Result<Vec<Schema>>>()?;
-            Ok(Schema::AnyOf {
-                options: new_options,
-            })
-        }
+        (Schema::OneOf { .. } | Schema::AnyOf { .. }, _) => {
+            merge_oneof_anyof((schemas.0, &Schema::AnyOf { options: vec![schemas.1.to_owned()] }))
+        },
+        (_, Schema::OneOf { .. } | Schema::AnyOf { .. }) => {
+            merge_oneof_anyof((&Schema::AnyOf { options: vec![schemas.0.to_owned()] }, schemas.1))
+        },
         (Schema::Null, Schema::Null) => Ok(Schema::Null),
         (Schema::Boolean, Schema::Boolean) => Ok(Schema::Boolean),
         (
@@ -498,7 +476,7 @@ fn merge_two(schemas: (&Schema, &Schema)) -> Result<Schema> {
             },
         ) => {
             let mut new_props = IndexMap::new();
-            for (key) in props1.keys().chain(props2.keys()) {
+            for key in props1.keys().chain(props2.keys()) {
                 let new_schema = match (props1.get(key), props2.get(key), add1, add2) {
                     (Some(schema1), Some(schema2), _, _) => merge_two((schema1, schema2))?,
                     (Some(schema1), None, _, Some(add)) => merge_two((schema1, &add))?,
@@ -604,7 +582,6 @@ fn merge_oneof_anyof(schemas: (&Schema, &Schema)) -> Result<Schema> {
 
 mod test {
     use super::*;
-    use regex_syntax::ast::print;
     use serde_json::json;
 
     #[test]
@@ -633,6 +610,9 @@ mod test {
                     "minimum": 10,
                 },
             ],
+            "allOf" : [
+                {"type": "string"}
+            ],
             "pattern": "^[a-z]+$",
             "maximum": 11
         });

From e462084221618485b9ae5cfec6cb0c6594da1d91 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Wed, 13 Nov 2024 18:20:44 -0800
Subject: [PATCH 32/73] silence for now

---
 parser/src/json/normalize.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index 8020f1d..8bf9c1b 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -50,6 +50,7 @@ enum Schema {
     OneOf {
         options: Vec<Schema>,
     },
+    #[allow(dead_code)]
     Ref(Box<Schema>),
 }
 

From 6c694ad7a493758e4d42a45a953c2c4fe823b298 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Wed, 13 Nov 2024 21:18:27 -0800
Subject: [PATCH 33/73] start cleaning up a bit

---
 parser/src/json/normalize.rs | 429 ++++++++++++++++++++---------------
 1 file changed, 252 insertions(+), 177 deletions(-)

diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index 8bf9c1b..1212fb3 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -2,8 +2,8 @@ use anyhow::{anyhow, bail, Result};
 use indexmap::IndexMap;
 use serde_json::{Map, Value};
 
-const TYPES: [&str; 7] = [
-    "null", "boolean", "integer", "number", "string", "array", "object",
+const TYPES: [&str; 6] = [
+    "null", "boolean", "number", "string", "array", "object",
 ];
 
 #[derive(Debug, PartialEq, Clone)]
@@ -54,173 +54,244 @@ enum Schema {
     Ref(Box<Schema>),
 }
 
-fn normalize(schema: &Value) -> Result<Schema> {
-    match schema.as_bool() {
-        Some(true) => return Ok(Schema::Any),
-        Some(false) => {
-            return Ok(Schema::Unsatisfiable {
-                reason: "schema is false".to_string(),
-            })
+impl Schema {
+    fn normalize(self) -> Result<Schema> {
+        match self {
+            Schema::Any => Ok(self),
+            Schema::Unsatisfiable { .. } => Ok(self),
+            Schema::Null => Ok(self),
+            Schema::Boolean => Ok(self),
+            Schema::Number { .. } => {
+                // TODO: validation logic, maybe returning Schema::Unsatisfiable
+                Ok(self)
+            },
+            Schema::String { .. } => {
+                // TODO: validation logic, maybe returning Schema::Unsatisfiable
+                Ok(self)
+            },
+            Schema::Array { min_items, max_items, prefix_items, items } => {
+                // TODO: validation logic, maybe returning Schema::Unsatisfiable
+                Ok(Schema::Array {
+                    min_items,
+                    max_items,
+                    prefix_items: prefix_items
+                        .into_iter()
+                        .map(|v| v.normalize())
+                        .collect::<Result<Vec<_>>>()?,
+                    items: items
+                        .map(|v| v.normalize().map(Box::new))
+                        .transpose()?
+                })
+            },
+            Schema::Object { properties, additional_properties, required } => {
+                // TODO: validation logic, maybe returning Schema::Unsatisfiable
+                Ok(Schema::Object { 
+                    properties: properties
+                        .into_iter()
+                        .map(|(k, v)| {
+                            v.normalize()
+                            .map(|v| (k, v))
+                        })
+                        .collect::<Result<IndexMap<_, _>>>()?,
+                    additional_properties: additional_properties
+                        .map(|v| v.normalize().map(Box::new))
+                        .transpose()?,
+                    required: required
+                })
+            },
+            Schema::Const { .. } => Ok(self),
+            Schema::Enum { options } => {
+                if options.is_empty() {
+                    return Ok(Schema::Unsatisfiable {
+                        reason: "enum is empty".to_string(),
+                    });
+                }
+                Ok(Schema::Enum { options })
+            },
+            Schema::AnyOf { options } => {
+                if options.is_empty() {
+                    return Ok(Schema::Unsatisfiable {
+                        reason: "anyOf is empty".to_string(),
+                    });
+                }
+                let mut unsats = Vec::new();
+                let mut valid = Vec::new();
+                for option in options {
+                    let normed = option.normalize()?;
+                    match normed {
+                        Schema::Unsatisfiable { .. } => unsats.push(normed),
+                        // Flatten nested anyOfs
+                        Schema::AnyOf { options: nested } => valid.extend(nested),
+                        _ => valid.push(normed),
+                    }
+                };
+                if valid.is_empty() {
+                    // Return the first unsatisfiable schema for debug-ability
+                    return Ok(unsats.swap_remove(0))
+                }
+                if valid.len() == 1 {
+                    return Ok(valid.swap_remove(0))
+                }
+                Ok(Schema::AnyOf { options: valid })
+            },
+            Schema::OneOf { options } => {
+                if options.is_empty() {
+                    return Ok(Schema::Unsatisfiable {
+                        reason: "oneOf is empty".to_string(),
+                    });
+                }
+                let mut unsats = Vec::new();
+                let mut valid = Vec::new();
+                for option in options {
+                    let normed = option.normalize()?;
+                    match normed {
+                        Schema::Unsatisfiable { .. } => unsats.push(normed),
+                        // Flatten nested oneOfs: (A⊕B)⊕(C⊕D) = A⊕B⊕C⊕D
+                        Schema::OneOf { options: nested } => valid.extend(nested),
+                        _ => valid.push(normed),
+                    }
+                };
+                if valid.is_empty() {
+                    // Return the first unsatisfiable schema for debug-ability
+                    return Ok(unsats.swap_remove(0))
+                }
+                if valid.len() == 1 {
+                    return Ok(valid.swap_remove(0))
+                }
+                Ok(Schema::OneOf { options: valid })
+            },
+            // TODO: ?
+            Schema::Ref(..) => Ok(self)
         }
-        None => {}
-    }
-
-    // Get the schema as an object
-    // TODO: validate against metaschema & check for unimplemented keys
-    let schema = schema
-        .as_object()
-        .ok_or_else(|| anyhow!("schema must be an object or boolean"))?;
-
-    if schema.is_empty() {
-        // TODO: should be ok to have ignored keys here
-        return Ok(Schema::Any);
     }
+}
 
-    if let Some(instance) = schema.get("const") {
-        // TODO: validate the instance against the schema, maybe returning Schema::Unsatisfiable
-        return Ok(Schema::Const {
-            value: instance.clone(),
-        });
-    }
+impl TryFrom<Value> for Schema {
+    type Error = anyhow::Error;
 
-    if let Some(instances) = schema.get("enum") {
-        let instances = instances
-            .as_array()
-            .ok_or_else(|| anyhow!("enum must be an array"))?;
-        if instances.is_empty() {
-            return Ok(Schema::Unsatisfiable {
-                reason: "enum is empty".to_string(),
+    fn try_from(value: Value) -> Result<Self, Self::Error> {
+        if let Some(b) = value.as_bool() {
+            return Ok({
+                if b {
+                    Schema::Any
+                } else {
+                    Schema::Unsatisfiable {
+                        reason: "schema is false".to_string(),
+                    }
+                }
             });
         }
-        // TODO: validate the instances against the schema, maybe returning Schema::Unsatisfiable
-        return Ok(Schema::Enum {
-            options: instances.clone(),
-        });
-    }
 
-    // Make a mutable copy of the schema so we can modify it
-    let mut schema = schema.clone();
+        // Get the schema as an object
+        // TODO: validate against metaschema & check for unimplemented keys
+        let schemadict = value
+            .as_object()
+            .ok_or_else(|| anyhow!("schema must be an object or boolean"))?;
 
-    if let Some(all_of) = schema.remove("allOf") {
-        let all_of = all_of
-            .as_array()
-            .ok_or_else(|| anyhow!("allOf must be an array"))?;
-        let siblings = normalize(&Value::Object(schema))?;
-        if all_of.is_empty() || matches!(siblings, Schema::Unsatisfiable { .. }) {
-            return Ok(siblings);
+        if schemadict.is_empty() {
+            // TODO: should be ok to have ignored keys here
+            return Ok(Schema::Any);
         }
-        let options = all_of
-            .iter()
-            .map(normalize)
-            .collect::<Result<Vec<Schema>>>()?;
-        let merged = merge(options.iter().chain(vec![&siblings]).collect())?;
-        return Ok(merged);
-    }
 
-    if let Some(any_of) = schema.remove("anyOf") {
-        let any_of = any_of
-            .as_array()
-            .ok_or_else(|| anyhow!("anyOf must be an array"))?;
-        let siblings = normalize(&Value::Object(schema))?;
-        if matches!(siblings, Schema::Unsatisfiable { .. }) {
-            return Ok(siblings);
-        }
-        if any_of.is_empty() {
-            return Ok(Schema::Unsatisfiable {
-                reason: "anyOf is empty".to_string(),
+        if let Some(instance) = schemadict.get("const") {
+            // TODO: validate the instance against the schema, maybe returning Schema::Unsatisfiable
+            return Ok(Schema::Const {
+                value: instance.clone(),
             });
         }
-        let options = any_of
-            .iter()
-            .filter_map(|value| match normalize(value) {
-                Ok(subschema) => {
-                    let merged = merge(vec![&subschema, &siblings]);
-                    match merged {
-                        Ok(Schema::Unsatisfiable { .. }) => None,
-                        _ => Some(merged),
-                    }
-                }
-                Err(_) => None,
-            })
-            .collect::<Result<Vec<Schema>>>()?;
-        if options.is_empty() {
-            return Ok(Schema::Unsatisfiable {
-                reason: "all anyOf options are unsatisfiable".to_string(),
+
+        if let Some(instances) = schemadict.get("enum") {
+            let instances = instances
+                .as_array()
+                .ok_or_else(|| anyhow!("enum must be an array"))?;
+            // TODO: validate the instances against the schema, maybe returning Schema::Unsatisfiable
+            return Ok(Schema::Enum {
+                options: instances.clone(),
             });
         }
-        return Ok(Schema::AnyOf { options });
-    }
 
-    // TODO: refactor to share code with anyOf
-    if let Some(one_of) = schema.remove("oneOf") {
-        let one_of = one_of
-            .as_array()
-            .ok_or_else(|| anyhow!("oneOf must be an array"))?;
-        let siblings = normalize(&Value::Object(schema))?;
-        if matches!(siblings, Schema::Unsatisfiable { .. }) {
-            return Ok(siblings);
-        }
-        if one_of.is_empty() {
-            return Ok(Schema::Unsatisfiable {
-                reason: "oneOf is empty".to_string(),
-            });
+        // Make a mutable copy of the schema so we can modify it
+        let mut schemadict = schemadict.clone();
+
+        if let Some(all_of) = schemadict.remove("allOf") {
+            let all_of = all_of
+                .as_array()
+                .ok_or_else(|| anyhow!("allOf must be an array"))?;
+            let siblings = Schema::try_from(Value::Object(schemadict))?;
+            // Short-circuit if schema is already unsatisfiable
+            if matches!(siblings, Schema::Unsatisfiable { .. }) {
+                return Ok(siblings);
+            }
+            let options = all_of
+                .iter()
+                .map(|value| Schema::try_from(value.to_owned()))
+                .collect::<Result<Vec<_>>>()?;
+            let merged = merge(options.iter().chain(vec![&siblings]).collect())?;
+            return Ok(merged);
         }
-        let options = one_of
-            .iter()
-            .filter_map(|value| match normalize(value) {
-                Ok(subschema) => {
-                    let merged = merge(vec![&subschema, &siblings]);
-                    match merged {
-                        Ok(Schema::Unsatisfiable { .. }) => None,
-                        _ => Some(merged),
-                    }
-                }
-                Err(_) => None,
-            })
-            .collect::<Result<Vec<Schema>>>()?;
-        if options.is_empty() {
-            return Ok(Schema::Unsatisfiable {
-                reason: "all oneOf options are unsatisfiable".to_string(),
-            });
+
+        if let Some(any_of) = schemadict.remove("anyOf") {
+            let any_of = any_of
+                .as_array()
+                .ok_or_else(|| anyhow!("anyOf must be an array"))?;
+            let siblings = Schema::try_from(Value::Object(schemadict))?;
+            // Short-circuit if schema is already unsatisfiable
+            if matches!(siblings, Schema::Unsatisfiable { .. }) {
+                return Ok(siblings);
+            }
+            let options = any_of
+                .iter()
+                .map(|value| Schema::try_from(value.to_owned()).and_then(|schema| merge_two(&schema, &siblings)))
+                .collect::<Result<Vec<_>>>()?;
+            return Ok(Schema::AnyOf { options });
         }
-        return Ok(Schema::OneOf { options });
-    }
 
-    if let Some(_) = schema.remove("$ref") {
-        bail!("Ref not implemented")
-    }
+        // TODO: refactor to share code with anyOf
+        if let Some(one_of) = schemadict.remove("oneOf") {
+            let one_of = one_of
+                .as_array()
+                .ok_or_else(|| anyhow!("oneOf must be an array"))?;
+            let siblings = Schema::try_from(Value::Object(schemadict))?;
+            // Short-circuit if schema is already unsatisfiable
+            if matches!(siblings, Schema::Unsatisfiable { .. }) {
+                return Ok(siblings);
+            }
+            let options = one_of
+                .iter()
+                .map(|value| Schema::try_from(value.to_owned()).and_then(|schema| merge_two(&schema, &siblings)))
+                .collect::<Result<Vec<_>>>()?;
+            return Ok(Schema::OneOf { options });
+        }
 
-    let types = match schema.remove("type") {
-        Some(Value::String(tp)) => {
-            return normalize_type(&schema, &tp);
+        if let Some(_) = schemadict.remove("$ref") {
+            bail!("Ref not implemented")
         }
-        Some(Value::Array(types)) => types
+
+        let types = match schemadict.remove("type") {
+            Some(Value::String(tp)) => {
+                return try_type(&schemadict, &tp);
+            }
+            Some(Value::Array(types)) => types
+                .iter()
+                .map(|tp| match tp.as_str() {
+                    Some(tp) => Ok(tp.to_string()),
+                    None => bail!("type must be a string"),
+                })
+                .collect::<Result<Vec<String>>>()?,
+            Some(_) => bail!("type must be a string or array"),
+            None => TYPES.iter().map(|s| s.to_string()).collect(),
+        };
+
+        let options = types
             .iter()
-            .map(|tp| match tp.as_str() {
-                Some(tp) => Ok(tp.to_string()),
-                None => bail!("type must be a string"),
-            })
-            .collect::<Result<Vec<String>>>()?,
-        Some(_) => bail!("type must be a string or array"),
-        None => TYPES.iter().map(|s| s.to_string()).collect(),
-    };
-    let options = types
-        .iter()
-        .map(|tp| normalize_type(&schema, &tp))
-        .filter_map(|result| match result {
-            Ok(Schema::Unsatisfiable { .. }) => None,
-            _ => Some(result),
-        })
-        .collect::<Result<Vec<Schema>>>()?;
-    if options.is_empty() {
-        return Ok(Schema::Unsatisfiable {
-            reason: "no valid types".to_string(),
-        });
+            .map(|tp| try_type(&schemadict, &tp))
+            .collect::<Result<Vec<Schema>>>()?;
+        Ok(Schema::AnyOf { options })
     }
-    Ok(Schema::AnyOf { options })
+
 }
 
-fn normalize_type(schema: &Map<String, Value>, tp: &str) -> Result<Schema> {
+fn try_type(schema: &Map<String, Value>, tp: &str) -> Result<Schema> {
     match tp {
         "null" => Ok(Schema::Null),
         "boolean" => Ok(Schema::Boolean),
@@ -271,12 +342,12 @@ fn normalize_type(schema: &Map<String, Value>, tp: &str) -> Result<Schema> {
                 .map(|items| {
                     items
                         .iter()
-                        .map(|item| normalize(item))
+                        .map(|item| Schema::try_from(item.to_owned()))
                         .collect::<Result<Vec<Schema>>>()
                 })
                 .transpose()?
                 .unwrap_or_default();
-            let items = schema.get("items").map(|v| normalize(v)).transpose()?;
+            let items = schema.get("items").map(|v| Schema::try_from(v.to_owned())).transpose()?;
             Ok(Schema::Array {
                 min_items,
                 max_items,
@@ -295,14 +366,14 @@ fn normalize_type(schema: &Map<String, Value>, tp: &str) -> Result<Schema> {
                 .map(|props| {
                     props
                         .iter()
-                        .map(|(k, v)| normalize(v).map(|schema| (k.clone(), schema)))
+                        .map(|(k, v)| Schema::try_from(v.to_owned()).map(|schema| (k.clone(), schema)))
                         .collect::<Result<IndexMap<String, Schema>>>()
                 })
                 .transpose()?
                 .unwrap_or_default();
             let additional_properties = schema
                 .get("additionalProperties")
-                .map(|v| normalize(v))
+                .map(|v| Schema::try_from(v.to_owned()))
                 .transpose()?;
             let required = schema
                 .get("required")
@@ -351,26 +422,26 @@ fn merge(schemas: Vec<&Schema>) -> Result<Schema> {
     // TODO: can we avoid cloning here?
     let mut merged = schemas[0].to_owned();
     for subschema in &schemas[1..] {
-        merged = merge_two((&merged, subschema))?;
+        merged = merge_two(&merged, subschema)?;
     }
     Ok(merged.to_owned())
 }
 
-fn merge_two(schemas: (&Schema, &Schema)) -> Result<Schema> {
-    match schemas {
-        (Schema::Any, _) => Ok(schemas.1.to_owned()),
-        (_, Schema::Any) => Ok(schemas.0.to_owned()),
-        (Schema::Unsatisfiable { .. }, _) => Ok(schemas.0.to_owned()),
-        (_, Schema::Unsatisfiable { .. }) => Ok(schemas.1.to_owned()),
+fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
+    match (schema0, schema1) {
+        (Schema::Any, _) => Ok(schema1.to_owned()),
+        (_, Schema::Any) => Ok(schema0.to_owned()),
+        (Schema::Unsatisfiable { .. }, _) => Ok(schema0.to_owned()),
+        (_, Schema::Unsatisfiable { .. }) => Ok(schema1.to_owned()),
         (
             Schema::OneOf { .. } | Schema::AnyOf { .. },
             Schema::OneOf { .. } | Schema::AnyOf { .. },
-        ) => merge_oneof_anyof(schemas),
+        ) => merge_oneof_anyof(schema0, schema1),
         (Schema::OneOf { .. } | Schema::AnyOf { .. }, _) => {
-            merge_oneof_anyof((schemas.0, &Schema::AnyOf { options: vec![schemas.1.to_owned()] }))
+            merge_oneof_anyof(schema0, &Schema::AnyOf { options: vec![schema1.to_owned()] })
         },
         (_, Schema::OneOf { .. } | Schema::AnyOf { .. }) => {
-            merge_oneof_anyof((&Schema::AnyOf { options: vec![schemas.0.to_owned()] }, schemas.1))
+            merge_oneof_anyof(&Schema::AnyOf { options: vec![schema0.to_owned()] }, schema1)
         },
         (Schema::Null, Schema::Null) => Ok(Schema::Null),
         (Schema::Boolean, Schema::Boolean) => Ok(Schema::Boolean),
@@ -394,7 +465,7 @@ fn merge_two(schemas: (&Schema, &Schema)) -> Result<Schema> {
             maximum: opt_min(*max1, *max2),
             exclusive_minimum: opt_max(*emin1, *emin2),
             exclusive_maximum: opt_min(*emax1, *emax2),
-            integer: *int1 && *int2,
+            integer: *int1 || *int2,
         }),
         (
             Schema::String {
@@ -455,13 +526,13 @@ fn merge_two(schemas: (&Schema, &Schema)) -> Result<Schema> {
             max_items: opt_min(*max1, *max2),
             prefix_items: zip_default(prefix1, prefix2, &Schema::Any)
                 .iter()
-                .map(|(item1, item2)| merge_two((item1, item2)))
+                .map(|(item1, item2)| merge_two(item1, item2))
                 .collect::<Result<Vec<Schema>>>()?,
             items: match (items1, items2) {
                 (None, None) => None,
                 (None, Some(item)) => Some(Box::new(*item.clone())),
                 (Some(item), None) => Some(Box::new(*item.clone())),
-                (Some(item1), Some(item2)) => Some(Box::new(merge_two((&item1, &item2))?)),
+                (Some(item1), Some(item2)) => Some(Box::new(merge_two(&item1, &item2)?)),
             },
         }),
         (
@@ -479,9 +550,9 @@ fn merge_two(schemas: (&Schema, &Schema)) -> Result<Schema> {
             let mut new_props = IndexMap::new();
             for key in props1.keys().chain(props2.keys()) {
                 let new_schema = match (props1.get(key), props2.get(key), add1, add2) {
-                    (Some(schema1), Some(schema2), _, _) => merge_two((schema1, schema2))?,
-                    (Some(schema1), None, _, Some(add)) => merge_two((schema1, &add))?,
-                    (None, Some(schema2), Some(add), _) => merge_two((&add, schema2))?,
+                    (Some(schema1), Some(schema2), _, _) => merge_two(schema1, schema2)?,
+                    (Some(schema1), None, _, Some(add)) => merge_two(schema1, &add)?,
+                    (None, Some(schema2), Some(add), _) => merge_two(&add, schema2)?,
                     (Some(schema1), None, _, None) => schema1.to_owned(),
                     (None, Some(schema2), None, _) => schema2.to_owned(),
                     (None, None, _, _) => bail!("should not happen"),
@@ -494,7 +565,7 @@ fn merge_two(schemas: (&Schema, &Schema)) -> Result<Schema> {
                     (None, None) => None,
                     (None, Some(add)) => Some(Box::new(*add.clone())),
                     (Some(add), None) => Some(Box::new(*add.clone())),
-                    (Some(add1), Some(add2)) => Some(Box::new(merge_two((&add1, &add2))?)),
+                    (Some(add1), Some(add2)) => Some(Box::new(merge_two(&add1, &add2)?)),
                 },
                 required: req1.iter().chain(req2.iter()).cloned().collect(),
             })
@@ -542,18 +613,18 @@ fn opt_min<T: PartialOrd>(a: Option<T>, b: Option<T>) -> Option<T> {
     }
 }
 
-fn merge_oneof_anyof(schemas: (&Schema, &Schema)) -> Result<Schema> {
-    let (schema1_options, schema2_options) = match schemas {
+fn merge_oneof_anyof(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
+    let (options0, options1) = match (schema0, schema1) {
         (
+            Schema::AnyOf { options: options0 } | Schema::OneOf { options: options0 },
             Schema::AnyOf { options: options1 } | Schema::OneOf { options: options1 },
-            Schema::AnyOf { options: options2 } | Schema::OneOf { options: options2 },
-        ) => (options1, options2),
+        ) => (options0, options1),
         _ => bail!("merge_oneof_anyof called with invalid schemas"),
     };
     let mut new_options = Vec::new();
-    for item1 in schema1_options {
-        for item2 in schema2_options {
-            let merged_item = merge(vec![item1, item2])?;
+    for item0 in options0 {
+        for item1 in options1 {
+            let merged_item = merge(vec![item0, item1])?;
             new_options.push(merged_item);
         }
     }
@@ -571,7 +642,7 @@ fn merge_oneof_anyof(schemas: (&Schema, &Schema)) -> Result<Schema> {
     }
 
     // OneOf takes precedence over AnyOf
-    if matches!(schemas.0, Schema::OneOf { .. }) || matches!(schemas.1, Schema::OneOf { .. }) {
+    if matches!(schema0, Schema::OneOf { .. }) || matches!(schema1, Schema::OneOf { .. }) {
         return Ok(Schema::OneOf {
             options: valid_options,
         });
@@ -594,7 +665,9 @@ mod test {
             "format": "email",
             "minimum": 10,
         });
-        let schema = normalize(&schema).unwrap();
+        let schema = Schema::try_from(schema).unwrap();
+        println!("{:?}", schema);
+        let schema = schema.normalize().unwrap();
         println!("{:?}", schema);
     }
 
@@ -612,12 +685,14 @@ mod test {
                 },
             ],
             "allOf" : [
-                {"type": "string"}
+                {"type": "integer"}
             ],
             "pattern": "^[a-z]+$",
             "maximum": 11
         });
-        let schema = normalize(&schema).unwrap();
+        let schema = Schema::try_from(schema).unwrap();
+        println!("{:?}", schema);
+        let schema = schema.normalize().unwrap();
         println!("{:?}", schema);
     }
 }

From 7f2a74e1a2dca1c44289fbb0660c9235e367b82c Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Wed, 13 Nov 2024 22:23:51 -0800
Subject: [PATCH 34/73] delete code!

---
 parser/src/json/normalize.rs | 57 +++++++-----------------------------
 1 file changed, 10 insertions(+), 47 deletions(-)

diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index 1212fb3..3f85358 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -433,15 +433,17 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
         (_, Schema::Any) => Ok(schema0.to_owned()),
         (Schema::Unsatisfiable { .. }, _) => Ok(schema0.to_owned()),
         (_, Schema::Unsatisfiable { .. }) => Ok(schema1.to_owned()),
-        (
-            Schema::OneOf { .. } | Schema::AnyOf { .. },
-            Schema::OneOf { .. } | Schema::AnyOf { .. },
-        ) => merge_oneof_anyof(schema0, schema1),
-        (Schema::OneOf { .. } | Schema::AnyOf { .. }, _) => {
-            merge_oneof_anyof(schema0, &Schema::AnyOf { options: vec![schema1.to_owned()] })
+        (Schema::OneOf { options }, _) => {
+            Ok(Schema::OneOf { options: options.iter().map(|opt| merge_two(opt, schema1)).collect::<Result<Vec<_>>>()? })
+        },
+        (_, Schema::OneOf { options }) => {
+            Ok(Schema::OneOf { options: options.iter().map(|opt| merge_two(schema0, opt)).collect::<Result<Vec<_>>>()? })
         },
-        (_, Schema::OneOf { .. } | Schema::AnyOf { .. }) => {
-            merge_oneof_anyof(&Schema::AnyOf { options: vec![schema0.to_owned()] }, schema1)
+        (Schema::AnyOf { options }, _) => {
+            Ok(Schema::AnyOf { options: options.iter().map(|opt| merge_two(opt, schema1)).collect::<Result<Vec<_>>>()? })
+        },
+        (_, Schema::AnyOf { options }) => {
+            Ok(Schema::AnyOf { options: options.iter().map(|opt| merge_two(schema0, opt)).collect::<Result<Vec<_>>>()? })
         },
         (Schema::Null, Schema::Null) => Ok(Schema::Null),
         (Schema::Boolean, Schema::Boolean) => Ok(Schema::Boolean),
@@ -613,45 +615,6 @@ fn opt_min<T: PartialOrd>(a: Option<T>, b: Option<T>) -> Option<T> {
     }
 }
 
-fn merge_oneof_anyof(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
-    let (options0, options1) = match (schema0, schema1) {
-        (
-            Schema::AnyOf { options: options0 } | Schema::OneOf { options: options0 },
-            Schema::AnyOf { options: options1 } | Schema::OneOf { options: options1 },
-        ) => (options0, options1),
-        _ => bail!("merge_oneof_anyof called with invalid schemas"),
-    };
-    let mut new_options = Vec::new();
-    for item0 in options0 {
-        for item1 in options1 {
-            let merged_item = merge(vec![item0, item1])?;
-            new_options.push(merged_item);
-        }
-    }
-    let valid_options = new_options
-        .iter()
-        .filter_map(|schema| match schema {
-            Schema::Unsatisfiable { .. } => None,
-            _ => Some(schema.to_owned()),
-        })
-        .collect::<Vec<Schema>>();
-    if valid_options.is_empty() {
-        // Return the first unsatisfiable schema for debug-ability
-        // TODO: think through ownership and don't return a clone if possible
-        return Ok(new_options[0].clone());
-    }
-
-    // OneOf takes precedence over AnyOf
-    if matches!(schema0, Schema::OneOf { .. }) || matches!(schema1, Schema::OneOf { .. }) {
-        return Ok(Schema::OneOf {
-            options: valid_options,
-        });
-    }
-    Ok(Schema::AnyOf {
-        options: valid_options,
-    })
-}
-
 mod test {
     use super::*;
     use serde_json::json;

From 42f526d0da38dfdd65ab097f1b5535475f249139 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Wed, 13 Nov 2024 22:24:04 -0800
Subject: [PATCH 35/73] testish

---
 parser/src/json/normalize.rs | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index 3f85358..4be1880 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -658,4 +658,35 @@ mod test {
         let schema = schema.normalize().unwrap();
         println!("{:?}", schema);
     }
+
+    #[test]
+    fn test_normalize_3() {
+        let schema = json!({
+            "anyOf": [
+                {
+                    "type": "string",
+                    "minLength": 1,
+                },
+                {
+                    "type": "number",
+                    "minimum": 10,
+                },
+            ],
+            "oneOf": [
+                {
+                    "type": ["string"],
+                    "maxLength": 1,
+                },
+                {
+                    "type": "number",
+                    "maximum": 11,
+                },
+            ],
+            "pattern": "^[a-z]+$",
+        });
+        let schema = Schema::try_from(schema).unwrap();
+        println!("{:?}", schema);
+        let schema = schema.normalize().unwrap();
+        println!("{:?}", schema);
+    }
 }

From 0279fc570aab93da0e76a0db10ab71d6b7f6d389 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Wed, 13 Nov 2024 22:27:51 -0800
Subject: [PATCH 36/73] normalize in try_from

---
 parser/src/json/normalize.rs | 217 +++++++++++++++++------------------
 1 file changed, 107 insertions(+), 110 deletions(-)

diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index 4be1880..3d7fe92 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -170,125 +170,128 @@ impl TryFrom<Value> for Schema {
     type Error = anyhow::Error;
 
     fn try_from(value: Value) -> Result<Self, Self::Error> {
-        if let Some(b) = value.as_bool() {
-            return Ok({
-                if b {
-                    Schema::Any
-                } else {
-                    Schema::Unsatisfiable {
-                        reason: "schema is false".to_string(),
-                    }
+        schema_from_value_inner(value)?.normalize()
+    }
+}
+
+fn schema_from_value_inner(value: Value) -> Result<Schema> {
+    if let Some(b) = value.as_bool() {
+        return Ok({
+            if b {
+                Schema::Any
+            } else {
+                Schema::Unsatisfiable {
+                    reason: "schema is false".to_string(),
                 }
-            });
-        }
+            }
+        });
+    }
 
-        // Get the schema as an object
-        // TODO: validate against metaschema & check for unimplemented keys
-        let schemadict = value
-            .as_object()
-            .ok_or_else(|| anyhow!("schema must be an object or boolean"))?;
+    // Get the schema as an object
+    // TODO: validate against metaschema & check for unimplemented keys
+    let schemadict = value
+        .as_object()
+        .ok_or_else(|| anyhow!("schema must be an object or boolean"))?;
 
-        if schemadict.is_empty() {
-            // TODO: should be ok to have ignored keys here
-            return Ok(Schema::Any);
-        }
+    if schemadict.is_empty() {
+        // TODO: should be ok to have ignored keys here
+        return Ok(Schema::Any);
+    }
 
-        if let Some(instance) = schemadict.get("const") {
-            // TODO: validate the instance against the schema, maybe returning Schema::Unsatisfiable
-            return Ok(Schema::Const {
-                value: instance.clone(),
-            });
-        }
+    if let Some(instance) = schemadict.get("const") {
+        // TODO: validate the instance against the schema, maybe returning Schema::Unsatisfiable
+        return Ok(Schema::Const {
+            value: instance.clone(),
+        });
+    }
 
-        if let Some(instances) = schemadict.get("enum") {
-            let instances = instances
-                .as_array()
-                .ok_or_else(|| anyhow!("enum must be an array"))?;
-            // TODO: validate the instances against the schema, maybe returning Schema::Unsatisfiable
-            return Ok(Schema::Enum {
-                options: instances.clone(),
-            });
-        }
+    if let Some(instances) = schemadict.get("enum") {
+        let instances = instances
+            .as_array()
+            .ok_or_else(|| anyhow!("enum must be an array"))?;
+        // TODO: validate the instances against the schema, maybe returning Schema::Unsatisfiable
+        return Ok(Schema::Enum {
+            options: instances.clone(),
+        });
+    }
 
-        // Make a mutable copy of the schema so we can modify it
-        let mut schemadict = schemadict.clone();
+    // Make a mutable copy of the schema so we can modify it
+    let mut schemadict = schemadict.clone();
 
-        if let Some(all_of) = schemadict.remove("allOf") {
-            let all_of = all_of
-                .as_array()
-                .ok_or_else(|| anyhow!("allOf must be an array"))?;
-            let siblings = Schema::try_from(Value::Object(schemadict))?;
-            // Short-circuit if schema is already unsatisfiable
-            if matches!(siblings, Schema::Unsatisfiable { .. }) {
-                return Ok(siblings);
-            }
-            let options = all_of
-                .iter()
-                .map(|value| Schema::try_from(value.to_owned()))
-                .collect::<Result<Vec<_>>>()?;
-            let merged = merge(options.iter().chain(vec![&siblings]).collect())?;
-            return Ok(merged);
-        }
-
-        if let Some(any_of) = schemadict.remove("anyOf") {
-            let any_of = any_of
-                .as_array()
-                .ok_or_else(|| anyhow!("anyOf must be an array"))?;
-            let siblings = Schema::try_from(Value::Object(schemadict))?;
-            // Short-circuit if schema is already unsatisfiable
-            if matches!(siblings, Schema::Unsatisfiable { .. }) {
-                return Ok(siblings);
-            }
-            let options = any_of
-                .iter()
-                .map(|value| Schema::try_from(value.to_owned()).and_then(|schema| merge_two(&schema, &siblings)))
-                .collect::<Result<Vec<_>>>()?;
-            return Ok(Schema::AnyOf { options });
+    if let Some(all_of) = schemadict.remove("allOf") {
+        let all_of = all_of
+            .as_array()
+            .ok_or_else(|| anyhow!("allOf must be an array"))?;
+        let siblings = Schema::try_from(Value::Object(schemadict))?;
+        // Short-circuit if schema is already unsatisfiable
+        if matches!(siblings, Schema::Unsatisfiable { .. }) {
+            return Ok(siblings);
         }
+        let options = all_of
+            .iter()
+            .map(|value| Schema::try_from(value.to_owned()))
+            .collect::<Result<Vec<_>>>()?;
+        let merged = merge(options.iter().chain(vec![&siblings]).collect())?;
+        return Ok(merged);
+    }
 
-        // TODO: refactor to share code with anyOf
-        if let Some(one_of) = schemadict.remove("oneOf") {
-            let one_of = one_of
-                .as_array()
-                .ok_or_else(|| anyhow!("oneOf must be an array"))?;
-            let siblings = Schema::try_from(Value::Object(schemadict))?;
-            // Short-circuit if schema is already unsatisfiable
-            if matches!(siblings, Schema::Unsatisfiable { .. }) {
-                return Ok(siblings);
-            }
-            let options = one_of
-                .iter()
-                .map(|value| Schema::try_from(value.to_owned()).and_then(|schema| merge_two(&schema, &siblings)))
-                .collect::<Result<Vec<_>>>()?;
-            return Ok(Schema::OneOf { options });
+    if let Some(any_of) = schemadict.remove("anyOf") {
+        let any_of = any_of
+            .as_array()
+            .ok_or_else(|| anyhow!("anyOf must be an array"))?;
+        let siblings = Schema::try_from(Value::Object(schemadict))?;
+        // Short-circuit if schema is already unsatisfiable
+        if matches!(siblings, Schema::Unsatisfiable { .. }) {
+            return Ok(siblings);
         }
+        let options = any_of
+            .iter()
+            .map(|value| Schema::try_from(value.to_owned()).and_then(|schema| merge_two(&schema, &siblings)))
+            .collect::<Result<Vec<_>>>()?;
+        return Ok(Schema::AnyOf { options });
+    }
 
-        if let Some(_) = schemadict.remove("$ref") {
-            bail!("Ref not implemented")
+    // TODO: refactor to share code with anyOf
+    if let Some(one_of) = schemadict.remove("oneOf") {
+        let one_of = one_of
+            .as_array()
+            .ok_or_else(|| anyhow!("oneOf must be an array"))?;
+        let siblings = Schema::try_from(Value::Object(schemadict))?;
+        // Short-circuit if schema is already unsatisfiable
+        if matches!(siblings, Schema::Unsatisfiable { .. }) {
+            return Ok(siblings);
         }
+        let options = one_of
+            .iter()
+            .map(|value| Schema::try_from(value.to_owned()).and_then(|schema| merge_two(&schema, &siblings)))
+            .collect::<Result<Vec<_>>>()?;
+        return Ok(Schema::OneOf { options });
+    }
 
-        let types = match schemadict.remove("type") {
-            Some(Value::String(tp)) => {
-                return try_type(&schemadict, &tp);
-            }
-            Some(Value::Array(types)) => types
-                .iter()
-                .map(|tp| match tp.as_str() {
-                    Some(tp) => Ok(tp.to_string()),
-                    None => bail!("type must be a string"),
-                })
-                .collect::<Result<Vec<String>>>()?,
-            Some(_) => bail!("type must be a string or array"),
-            None => TYPES.iter().map(|s| s.to_string()).collect(),
-        };
+    if let Some(_) = schemadict.remove("$ref") {
+        bail!("Ref not implemented")
+    }
 
-        let options = types
+    let types = match schemadict.remove("type") {
+        Some(Value::String(tp)) => {
+            return try_type(&schemadict, &tp);
+        }
+        Some(Value::Array(types)) => types
             .iter()
-            .map(|tp| try_type(&schemadict, &tp))
-            .collect::<Result<Vec<Schema>>>()?;
-        Ok(Schema::AnyOf { options })
-    }
+            .map(|tp| match tp.as_str() {
+                Some(tp) => Ok(tp.to_string()),
+                None => bail!("type must be a string"),
+            })
+            .collect::<Result<Vec<String>>>()?,
+        Some(_) => bail!("type must be a string or array"),
+        None => TYPES.iter().map(|s| s.to_string()).collect(),
+    };
 
+    let options = types
+        .iter()
+        .map(|tp| try_type(&schemadict, &tp))
+        .collect::<Result<Vec<Schema>>>()?;
+    Ok(Schema::AnyOf { options })
 }
 
 fn try_type(schema: &Map<String, Value>, tp: &str) -> Result<Schema> {
@@ -630,8 +633,6 @@ mod test {
         });
         let schema = Schema::try_from(schema).unwrap();
         println!("{:?}", schema);
-        let schema = schema.normalize().unwrap();
-        println!("{:?}", schema);
     }
 
     #[test]
@@ -655,8 +656,6 @@ mod test {
         });
         let schema = Schema::try_from(schema).unwrap();
         println!("{:?}", schema);
-        let schema = schema.normalize().unwrap();
-        println!("{:?}", schema);
     }
 
     #[test]
@@ -674,7 +673,7 @@ mod test {
             ],
             "oneOf": [
                 {
-                    "type": ["string"],
+                    "type": ["string", "integer"],
                     "maxLength": 1,
                 },
                 {
@@ -686,7 +685,5 @@ mod test {
         });
         let schema = Schema::try_from(schema).unwrap();
         println!("{:?}", schema);
-        let schema = schema.normalize().unwrap();
-        println!("{:?}", schema);
     }
 }

From 64607e411c9b8d70cd6e9b87840bade7a7d3ae1f Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Wed, 13 Nov 2024 22:28:48 -0800
Subject: [PATCH 37/73] fmt normalize

---
 parser/src/json/normalize.rs | 111 +++++++++++++++++++++--------------
 1 file changed, 67 insertions(+), 44 deletions(-)

diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index 3d7fe92..0b8b693 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -2,9 +2,7 @@ use anyhow::{anyhow, bail, Result};
 use indexmap::IndexMap;
 use serde_json::{Map, Value};
 
-const TYPES: [&str; 6] = [
-    "null", "boolean", "number", "string", "array", "object",
-];
+const TYPES: [&str; 6] = ["null", "boolean", "number", "string", "array", "object"];
 
 #[derive(Debug, PartialEq, Clone)]
 enum Schema {
@@ -64,12 +62,17 @@ impl Schema {
             Schema::Number { .. } => {
                 // TODO: validation logic, maybe returning Schema::Unsatisfiable
                 Ok(self)
-            },
+            }
             Schema::String { .. } => {
                 // TODO: validation logic, maybe returning Schema::Unsatisfiable
                 Ok(self)
-            },
-            Schema::Array { min_items, max_items, prefix_items, items } => {
+            }
+            Schema::Array {
+                min_items,
+                max_items,
+                prefix_items,
+                items,
+            } => {
                 // TODO: validation logic, maybe returning Schema::Unsatisfiable
                 Ok(Schema::Array {
                     min_items,
@@ -78,27 +81,26 @@ impl Schema {
                         .into_iter()
                         .map(|v| v.normalize())
                         .collect::<Result<Vec<_>>>()?,
-                    items: items
-                        .map(|v| v.normalize().map(Box::new))
-                        .transpose()?
+                    items: items.map(|v| v.normalize().map(Box::new)).transpose()?,
                 })
-            },
-            Schema::Object { properties, additional_properties, required } => {
+            }
+            Schema::Object {
+                properties,
+                additional_properties,
+                required,
+            } => {
                 // TODO: validation logic, maybe returning Schema::Unsatisfiable
-                Ok(Schema::Object { 
+                Ok(Schema::Object {
                     properties: properties
                         .into_iter()
-                        .map(|(k, v)| {
-                            v.normalize()
-                            .map(|v| (k, v))
-                        })
+                        .map(|(k, v)| v.normalize().map(|v| (k, v)))
                         .collect::<Result<IndexMap<_, _>>>()?,
                     additional_properties: additional_properties
                         .map(|v| v.normalize().map(Box::new))
                         .transpose()?,
-                    required: required
+                    required: required,
                 })
-            },
+            }
             Schema::Const { .. } => Ok(self),
             Schema::Enum { options } => {
                 if options.is_empty() {
@@ -107,7 +109,7 @@ impl Schema {
                     });
                 }
                 Ok(Schema::Enum { options })
-            },
+            }
             Schema::AnyOf { options } => {
                 if options.is_empty() {
                     return Ok(Schema::Unsatisfiable {
@@ -124,16 +126,16 @@ impl Schema {
                         Schema::AnyOf { options: nested } => valid.extend(nested),
                         _ => valid.push(normed),
                     }
-                };
+                }
                 if valid.is_empty() {
                     // Return the first unsatisfiable schema for debug-ability
-                    return Ok(unsats.swap_remove(0))
+                    return Ok(unsats.swap_remove(0));
                 }
                 if valid.len() == 1 {
-                    return Ok(valid.swap_remove(0))
+                    return Ok(valid.swap_remove(0));
                 }
                 Ok(Schema::AnyOf { options: valid })
-            },
+            }
             Schema::OneOf { options } => {
                 if options.is_empty() {
                     return Ok(Schema::Unsatisfiable {
@@ -150,18 +152,18 @@ impl Schema {
                         Schema::OneOf { options: nested } => valid.extend(nested),
                         _ => valid.push(normed),
                     }
-                };
+                }
                 if valid.is_empty() {
                     // Return the first unsatisfiable schema for debug-ability
-                    return Ok(unsats.swap_remove(0))
+                    return Ok(unsats.swap_remove(0));
                 }
                 if valid.len() == 1 {
-                    return Ok(valid.swap_remove(0))
+                    return Ok(valid.swap_remove(0));
                 }
                 Ok(Schema::OneOf { options: valid })
-            },
+            }
             // TODO: ?
-            Schema::Ref(..) => Ok(self)
+            Schema::Ref(..) => Ok(self),
         }
     }
 }
@@ -246,7 +248,9 @@ fn schema_from_value_inner(value: Value) -> Result<Schema> {
         }
         let options = any_of
             .iter()
-            .map(|value| Schema::try_from(value.to_owned()).and_then(|schema| merge_two(&schema, &siblings)))
+            .map(|value| {
+                Schema::try_from(value.to_owned()).and_then(|schema| merge_two(&schema, &siblings))
+            })
             .collect::<Result<Vec<_>>>()?;
         return Ok(Schema::AnyOf { options });
     }
@@ -263,7 +267,9 @@ fn schema_from_value_inner(value: Value) -> Result<Schema> {
         }
         let options = one_of
             .iter()
-            .map(|value| Schema::try_from(value.to_owned()).and_then(|schema| merge_two(&schema, &siblings)))
+            .map(|value| {
+                Schema::try_from(value.to_owned()).and_then(|schema| merge_two(&schema, &siblings))
+            })
             .collect::<Result<Vec<_>>>()?;
         return Ok(Schema::OneOf { options });
     }
@@ -350,7 +356,10 @@ fn try_type(schema: &Map<String, Value>, tp: &str) -> Result<Schema> {
                 })
                 .transpose()?
                 .unwrap_or_default();
-            let items = schema.get("items").map(|v| Schema::try_from(v.to_owned())).transpose()?;
+            let items = schema
+                .get("items")
+                .map(|v| Schema::try_from(v.to_owned()))
+                .transpose()?;
             Ok(Schema::Array {
                 min_items,
                 max_items,
@@ -369,7 +378,9 @@ fn try_type(schema: &Map<String, Value>, tp: &str) -> Result<Schema> {
                 .map(|props| {
                     props
                         .iter()
-                        .map(|(k, v)| Schema::try_from(v.to_owned()).map(|schema| (k.clone(), schema)))
+                        .map(|(k, v)| {
+                            Schema::try_from(v.to_owned()).map(|schema| (k.clone(), schema))
+                        })
                         .collect::<Result<IndexMap<String, Schema>>>()
                 })
                 .transpose()?
@@ -436,18 +447,30 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
         (_, Schema::Any) => Ok(schema0.to_owned()),
         (Schema::Unsatisfiable { .. }, _) => Ok(schema0.to_owned()),
         (_, Schema::Unsatisfiable { .. }) => Ok(schema1.to_owned()),
-        (Schema::OneOf { options }, _) => {
-            Ok(Schema::OneOf { options: options.iter().map(|opt| merge_two(opt, schema1)).collect::<Result<Vec<_>>>()? })
-        },
-        (_, Schema::OneOf { options }) => {
-            Ok(Schema::OneOf { options: options.iter().map(|opt| merge_two(schema0, opt)).collect::<Result<Vec<_>>>()? })
-        },
-        (Schema::AnyOf { options }, _) => {
-            Ok(Schema::AnyOf { options: options.iter().map(|opt| merge_two(opt, schema1)).collect::<Result<Vec<_>>>()? })
-        },
-        (_, Schema::AnyOf { options }) => {
-            Ok(Schema::AnyOf { options: options.iter().map(|opt| merge_two(schema0, opt)).collect::<Result<Vec<_>>>()? })
-        },
+        (Schema::OneOf { options }, _) => Ok(Schema::OneOf {
+            options: options
+                .iter()
+                .map(|opt| merge_two(opt, schema1))
+                .collect::<Result<Vec<_>>>()?,
+        }),
+        (_, Schema::OneOf { options }) => Ok(Schema::OneOf {
+            options: options
+                .iter()
+                .map(|opt| merge_two(schema0, opt))
+                .collect::<Result<Vec<_>>>()?,
+        }),
+        (Schema::AnyOf { options }, _) => Ok(Schema::AnyOf {
+            options: options
+                .iter()
+                .map(|opt| merge_two(opt, schema1))
+                .collect::<Result<Vec<_>>>()?,
+        }),
+        (_, Schema::AnyOf { options }) => Ok(Schema::AnyOf {
+            options: options
+                .iter()
+                .map(|opt| merge_two(schema0, opt))
+                .collect::<Result<Vec<_>>>()?,
+        }),
         (Schema::Null, Schema::Null) => Ok(Schema::Null),
         (Schema::Boolean, Schema::Boolean) => Ok(Schema::Boolean),
         (

From cf3fe91f30c7dc0ef33f7ddbeaf1c6ca6e66cf89 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Thu, 14 Nov 2024 09:57:58 -0800
Subject: [PATCH 38/73] fix merged prefixItems by applying other schema's items

---
 parser/src/json/normalize.rs | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index 0b8b693..370bc75 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -552,10 +552,15 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
         ) => Ok(Schema::Array {
             min_items: *min1.max(min2),
             max_items: opt_min(*max1, *max2),
-            prefix_items: zip_default(prefix1, prefix2, &Schema::Any)
-                .iter()
-                .map(|(item1, item2)| merge_two(item1, item2))
-                .collect::<Result<Vec<Schema>>>()?,
+            prefix_items: zip_default(
+                prefix1,
+                prefix2,
+                items2.as_deref().unwrap_or(&Schema::Any),
+                items1.as_deref().unwrap_or(&Schema::Any),
+            )
+            .iter()
+            .map(|(item1, item2)| merge_two(item1, item2))
+            .collect::<Result<Vec<Schema>>>()?,
             items: match (items1, items2) {
                 (None, None) => None,
                 (None, Some(item)) => Some(Box::new(*item.clone())),
@@ -605,9 +610,14 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
     }
 }
 
-fn zip_default<'a, T>(arr1: &'a [T], arr2: &'a [T], default: &'a T) -> Vec<(&'a T, &'a T)> {
-    let iter1 = arr1.iter().chain(std::iter::repeat(default));
-    let iter2 = arr2.iter().chain(std::iter::repeat(default));
+fn zip_default<'a, T>(
+    arr1: &'a [T],
+    arr2: &'a [T],
+    default1: &'a T,
+    default2: &'a T,
+) -> Vec<(&'a T, &'a T)> {
+    let iter1 = arr1.iter().chain(std::iter::repeat(default1));
+    let iter2 = arr2.iter().chain(std::iter::repeat(default2));
     iter1.zip(iter2).take(arr1.len().max(arr2.len())).collect()
 }
 

From 80f9ead2a37b28f7a73457e15989e02ee3fd14a2 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Thu, 14 Nov 2024 10:39:40 -0800
Subject: [PATCH 39/73] required as IndexSet

---
 parser/src/json/normalize.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index 370bc75..172488f 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -1,5 +1,5 @@
 use anyhow::{anyhow, bail, Result};
-use indexmap::IndexMap;
+use indexmap::{IndexMap, IndexSet};
 use serde_json::{Map, Value};
 
 const TYPES: [&str; 6] = ["null", "boolean", "number", "string", "array", "object"];
@@ -34,7 +34,7 @@ enum Schema {
     Object {
         properties: IndexMap<String, Schema>,
         additional_properties: Option<Box<Schema>>,
-        required: Vec<String>,
+        required: IndexSet<String>,
     },
     Const {
         value: Value,
@@ -411,7 +411,7 @@ fn try_type(schema: &Map<String, Value>, tp: &str) -> Result<Schema> {
             Ok(Schema::Object {
                 properties,
                 additional_properties: additional_properties.map(Box::new),
-                required,
+                required: IndexSet::from_iter(required),
             })
         }
         _ => bail!("Invalid type: {}", tp),
@@ -600,7 +600,7 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
                     (Some(add), None) => Some(Box::new(*add.clone())),
                     (Some(add1), Some(add2)) => Some(Box::new(merge_two(&add1, &add2)?)),
                 },
-                required: req1.iter().chain(req2.iter()).cloned().collect(),
+                required: req1.union(req2).cloned().collect()
             })
         }
         //TODO: get types for error message

From bdd7b46f9ad10b49558496d36083950777a459a5 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Thu, 14 Nov 2024 13:17:09 -0800
Subject: [PATCH 40/73] refactor and flesh out try_type

---
 parser/src/json/normalize.rs | 325 +++++++++++++++++++++++------------
 1 file changed, 214 insertions(+), 111 deletions(-)

diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index 172488f..8aa4884 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -4,6 +4,15 @@ use serde_json::{Map, Value};
 
 const TYPES: [&str; 6] = ["null", "boolean", "number", "string", "array", "object"];
 
+fn limited_str(node: &Value) -> String {
+    let s = node.to_string();
+    if s.len() > 100 {
+        format!("{}...", &s[..100])
+    } else {
+        s
+    }
+}
+
 #[derive(Debug, PartialEq, Clone)]
 enum Schema {
     Any,
@@ -304,120 +313,214 @@ fn try_type(schema: &Map<String, Value>, tp: &str) -> Result<Schema> {
     match tp {
         "null" => Ok(Schema::Null),
         "boolean" => Ok(Schema::Boolean),
-        "number" | "integer" => {
-            let minimum = schema.get("minimum").and_then(|v| v.as_f64());
-            let maximum = schema.get("maximum").and_then(|v| v.as_f64());
-            let exclusive_minimum = schema.get("exclusiveMinimum").and_then(|v| v.as_f64());
-            let exclusive_maximum = schema.get("exclusiveMaximum").and_then(|v| v.as_f64());
-            Ok(Schema::Number {
-                minimum,
-                maximum,
-                exclusive_minimum,
-                exclusive_maximum,
-                integer: tp == "integer",
-            })
-        }
-        "string" => {
-            let min_length = schema
-                .get("minLength")
-                .and_then(|v| v.as_u64())
-                .unwrap_or(0);
-            let max_length = schema.get("maxLength").and_then(|v| v.as_u64());
-            let pattern = schema
-                .get("pattern")
-                .and_then(|v| v.as_str())
-                .map(|s| s.to_string());
-            let format = schema
-                .get("format")
-                .and_then(|v| v.as_str())
-                .map(|s| s.to_string());
-            Ok(Schema::String {
-                min_length,
-                max_length,
-                pattern,
-                format,
-            })
-        }
-        "array" => {
-            let min_items = schema.get("minItems").and_then(|v| v.as_u64()).unwrap_or(0);
-            let max_items = schema.get("maxItems").and_then(|v| v.as_u64());
-            let prefix_items = schema
-                .get("prefixItems")
-                .map(|v| {
-                    v.as_array()
-                        .ok_or_else(|| anyhow!("prefixItems must be an array"))
-                })
-                .transpose()?
-                .map(|items| {
-                    items
-                        .iter()
-                        .map(|item| Schema::try_from(item.to_owned()))
-                        .collect::<Result<Vec<Schema>>>()
-                })
-                .transpose()?
-                .unwrap_or_default();
-            let items = schema
-                .get("items")
-                .map(|v| Schema::try_from(v.to_owned()))
-                .transpose()?;
-            Ok(Schema::Array {
-                min_items,
-                max_items,
-                prefix_items,
-                items: items.map(Box::new),
-            })
-        }
-        "object" => {
-            let properties = schema
-                .get("properties")
-                .map(|v| {
-                    v.as_object()
-                        .ok_or_else(|| anyhow!("properties must be an object"))
-                })
-                .transpose()?
-                .map(|props| {
-                    props
-                        .iter()
-                        .map(|(k, v)| {
-                            Schema::try_from(v.to_owned()).map(|schema| (k.clone(), schema))
-                        })
-                        .collect::<Result<IndexMap<String, Schema>>>()
-                })
-                .transpose()?
-                .unwrap_or_default();
-            let additional_properties = schema
-                .get("additionalProperties")
-                .map(|v| Schema::try_from(v.to_owned()))
-                .transpose()?;
-            let required = schema
-                .get("required")
-                .map(|v| {
-                    v.as_array()
-                        .ok_or_else(|| anyhow!("required must be an array"))
-                })
-                .transpose()?
-                .map(|items| {
-                    items
-                        .iter()
-                        .map(|item| {
-                            item.as_str()
-                                .ok_or_else(|| anyhow!("required items must be strings"))
-                                .map(|s| s.to_string())
-                        })
-                        .collect::<Result<Vec<String>>>()
-                })
-                .transpose()?
-                .unwrap_or_default();
-            Ok(Schema::Object {
-                properties,
-                additional_properties: additional_properties.map(Box::new),
-                required: IndexSet::from_iter(required),
-            })
-        }
+        "number" | "integer" => try_numeric(
+            schema.get("minimum"),
+            schema.get("maximum"),
+            schema.get("exclusiveMinimum"),
+            schema.get("exclusiveMaximum"),
+            tp == "integer",
+        ),
+        "string" => try_string(
+            schema.get("minLength"),
+            schema.get("maxLength"),
+            schema.get("pattern"),
+            schema.get("format"),
+        ),
+        "array" => try_array(
+            schema.get("minItems"),
+            schema.get("maxItems"),
+            schema.get("prefixItems"),
+            schema.get("items"),
+        ),
+        "object" => try_object(
+            schema.get("properties"),
+            schema.get("additionalProperties"),
+            schema.get("required"),
+        ),
         _ => bail!("Invalid type: {}", tp),
     }
 }
 
+fn try_numeric(
+    minimum: Option<&Value>,
+    maximum: Option<&Value>,
+    exclusive_minimum: Option<&Value>,
+    exclusive_maximum: Option<&Value>,
+    integer: bool,
+) -> Result<Schema> {
+    let minimum = match minimum {
+        None => None,
+        Some(val) => Some(
+            val.as_f64()
+                .ok_or_else(|| anyhow!("Expected f64 for 'minimum', got {}", limited_str(val)))?,
+        ),
+    };
+    let maximum = match maximum {
+        None => None,
+        Some(val) => Some(
+            val.as_f64()
+                .ok_or_else(|| anyhow!("Expected f64 for 'maximum', got {}", limited_str(val)))?,
+        ),
+    };
+    let exclusive_minimum = match exclusive_minimum {
+        // Draft4-style boolean values
+        None | Some(Value::Bool(false)) => None,
+        Some(Value::Bool(true)) => minimum,
+        // Draft2020-12-style numeric values
+        Some(value) => Some(value.as_f64().ok_or_else(|| {
+            anyhow!(
+                "Expected f64 for 'exclusiveMinimum', got {}",
+                limited_str(value)
+            )
+        })?),
+    };
+    let exclusive_maximum = match exclusive_maximum {
+        // Draft4-style boolean values
+        None | Some(Value::Bool(false)) => None,
+        Some(Value::Bool(true)) => maximum,
+        // Draft2020-12-style numeric values
+        Some(value) => Some(value.as_f64().ok_or_else(|| {
+            anyhow!(
+                "Expected f64 for 'exclusiveMaximum', got {}",
+                limited_str(value)
+            )
+        })?),
+    };
+    Ok(Schema::Number {
+        minimum,
+        maximum,
+        exclusive_minimum,
+        exclusive_maximum,
+        integer: integer,
+    })
+}
+
+fn try_string(
+    min_length: Option<&Value>,
+    max_length: Option<&Value>,
+    pattern: Option<&Value>,
+    format: Option<&Value>,
+) -> Result<Schema> {
+    let min_length = match min_length {
+        None => 0,
+        Some(val) => val
+            .as_u64()
+            .ok_or_else(|| anyhow!("Expected u64 for 'minLength', got {}", limited_str(val)))?,
+    };
+    let max_length = match max_length {
+        None => None,
+        Some(val) => Some(
+            val.as_u64()
+                .ok_or_else(|| anyhow!("Expected u64 for 'maxLength', got {}", limited_str(val)))?,
+        ),
+    };
+    let pattern = match pattern {
+        None => None,
+        Some(val) => Some(
+            val.as_str()
+                .ok_or_else(|| anyhow!("Expected string for 'pattern', got {}", limited_str(val)))?
+                .to_string(),
+        ),
+    };
+    let format = match format {
+        None => None,
+        Some(val) => Some(
+            val.as_str()
+                .ok_or_else(|| anyhow!("Expected string for 'format', got {}", limited_str(val)))?
+                .to_string(),
+        ),
+    };
+    Ok(Schema::String {
+        min_length,
+        max_length,
+        pattern,
+        format,
+    })
+}
+
+fn try_array(
+    min_items: Option<&Value>,
+    max_items: Option<&Value>,
+    prefix_items: Option<&Value>,
+    items: Option<&Value>,
+) -> Result<Schema> {
+    let min_items = match min_items {
+        None => 0,
+        Some(val) => val
+            .as_u64()
+            .ok_or_else(|| anyhow!("Expected u64 for 'minItems', got {}", limited_str(val)))?,
+    };
+    let max_items = match max_items {
+        None => None,
+        Some(val) => Some(
+            val.as_u64()
+                .ok_or_else(|| anyhow!("Expected u64 for 'maxItems', got {}", limited_str(val)))?,
+        ),
+    };
+    let prefix_items = match prefix_items {
+        None => vec![],
+        Some(val) => val
+            .as_array()
+            .ok_or_else(|| anyhow!("Expected array for 'prefixItems', got {}", limited_str(val)))?
+            .iter()
+            .map(|item| Schema::try_from(item.to_owned()))
+            .collect::<Result<Vec<Schema>>>()?,
+    };
+    let items = match items {
+        None => None,
+        Some(val) => Some(Box::new(Schema::try_from(val.to_owned())?)),
+    };
+    Ok(Schema::Array {
+        min_items,
+        max_items,
+        prefix_items,
+        items,
+    })
+}
+
+fn try_object(
+    properties: Option<&Value>,
+    additional_properties: Option<&Value>,
+    required: Option<&Value>,
+) -> Result<Schema> {
+    let properties = match properties {
+        None => IndexMap::new(),
+        Some(val) => val
+            .as_object()
+            .ok_or_else(|| anyhow!("Expected object for 'properties', got {}", limited_str(val)))?
+            .iter()
+            .map(|(k, v)| Schema::try_from(v.to_owned()).map(|v| (k.clone(), v)))
+            .collect::<Result<IndexMap<String, Schema>>>()?,
+    };
+    let additional_properties = match additional_properties {
+        None => None,
+        Some(val) => Some(Box::new(Schema::try_from(val.to_owned())?)),
+    };
+    let required = match required {
+        None => IndexSet::new(),
+        Some(val) => val
+            .as_array()
+            .ok_or_else(|| anyhow!("Expected array for 'required', got {}", limited_str(val)))?
+            .iter()
+            .map(|item| {
+                item.as_str()
+                    .ok_or_else(|| {
+                        anyhow!(
+                            "Expected string for 'required' item, got {}",
+                            limited_str(item)
+                        )
+                    })
+                    .map(|s| s.to_string())
+            })
+            .collect::<Result<IndexSet<String>>>()?,
+    };
+    Ok(Schema::Object {
+        properties,
+        additional_properties,
+        required,
+    })
+}
+
 fn merge(schemas: Vec<&Schema>) -> Result<Schema> {
     if schemas.is_empty() {
         bail!("merge called with empty list")
@@ -600,7 +703,7 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
                     (Some(add), None) => Some(Box::new(*add.clone())),
                     (Some(add1), Some(add2)) => Some(Box::new(merge_two(&add1, &add2)?)),
                 },
-                required: req1.union(req2).cloned().collect()
+                required: req1.union(req2).cloned().collect(),
             })
         }
         //TODO: get types for error message

From 91e9a6cac22273138b6149eac9b33e702fa6ac98 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Thu, 14 Nov 2024 13:30:56 -0800
Subject: [PATCH 41/73] fix merge in case that a schema is unsat

---
 parser/src/json/normalize.rs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index 8aa4884..364bd55 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -528,20 +528,20 @@ fn merge(schemas: Vec<&Schema>) -> Result<Schema> {
     if schemas.iter().all(|schema| matches!(schema, Schema::Any)) {
         return Ok(Schema::Any);
     }
-    if schemas
-        .iter()
-        .any(|schema| matches!(schema, Schema::Unsatisfiable { .. }))
-    {
+    if let Some(unsat) = schemas.iter().find(|schema| matches!(schema, Schema::Unsatisfiable { .. })) {
         // Return the first unsatisfiable schema for debug-ability
-        // TODO: think through ownership and don't return a clone if possible
-        return Ok(schemas[0].to_owned());
+        return Ok((*unsat).to_owned());
     }
     // TODO: can we avoid cloning here?
     let mut merged = schemas[0].to_owned();
     for subschema in &schemas[1..] {
         merged = merge_two(&merged, subschema)?;
+        if matches!(merged, Schema::Unsatisfiable { .. }) {
+            // Early exit if the schema is already unsatisfiable
+            break;
+        }
     }
-    Ok(merged.to_owned())
+    Ok(merged)
 }
 
 fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {

From 3c38375ca968b228d0d20fe55e97a8d10efeb92b Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Thu, 14 Nov 2024 13:43:01 -0800
Subject: [PATCH 42/73] simple ref without siblings

---
 parser/src/json/normalize.rs | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index 364bd55..51fccc9 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -58,7 +58,7 @@ enum Schema {
         options: Vec<Schema>,
     },
     #[allow(dead_code)]
-    Ref(Box<Schema>),
+    Ref(String),
 }
 
 impl Schema {
@@ -283,8 +283,17 @@ fn schema_from_value_inner(value: Value) -> Result<Schema> {
         return Ok(Schema::OneOf { options });
     }
 
-    if let Some(_) = schemadict.remove("$ref") {
-        bail!("Ref not implemented")
+    if let Some(reference) = schemadict.remove("$ref") {
+        let reference = reference
+            .as_str()
+            .ok_or_else(|| anyhow!("$ref must be a string, got {}", limited_str(&reference)))?
+            .to_string();
+        let siblings = Schema::try_from(Value::Object(schemadict))?;
+        // Short-circuit if schema is already unsatisfiable
+        if matches!(siblings, Schema::Unsatisfiable { .. }) {
+            return Ok(siblings);
+        }
+        return Ok(merge_two(&siblings, &Schema::Ref(reference))?);
     }
 
     let types = match schemadict.remove("type") {
@@ -302,6 +311,7 @@ fn schema_from_value_inner(value: Value) -> Result<Schema> {
         None => TYPES.iter().map(|s| s.to_string()).collect(),
     };
 
+    // Shouldn't need siblings here since we've already handled allOf, anyOf, oneOf, and $ref
     let options = types
         .iter()
         .map(|tp| try_type(&schemadict, &tp))
@@ -528,7 +538,10 @@ fn merge(schemas: Vec<&Schema>) -> Result<Schema> {
     if schemas.iter().all(|schema| matches!(schema, Schema::Any)) {
         return Ok(Schema::Any);
     }
-    if let Some(unsat) = schemas.iter().find(|schema| matches!(schema, Schema::Unsatisfiable { .. })) {
+    if let Some(unsat) = schemas
+        .iter()
+        .find(|schema| matches!(schema, Schema::Unsatisfiable { .. }))
+    {
         // Return the first unsatisfiable schema for debug-ability
         return Ok((*unsat).to_owned());
     }
@@ -550,6 +563,12 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
         (_, Schema::Any) => Ok(schema0.to_owned()),
         (Schema::Unsatisfiable { .. }, _) => Ok(schema0.to_owned()),
         (_, Schema::Unsatisfiable { .. }) => Ok(schema1.to_owned()),
+        (Schema::Ref(reference), _) => {
+            Err(anyhow!("$ref with siblings not implemented: {}", reference))
+        }
+        (_, Schema::Ref(reference)) => {
+            Err(anyhow!("$ref with siblings not implemented: {}", reference))
+        }
         (Schema::OneOf { options }, _) => Ok(Schema::OneOf {
             options: options
                 .iter()

From e3ef006bf78a26501a0b5a9b67ce3b59fa3176cd Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Thu, 14 Nov 2024 21:42:48 -0800
Subject: [PATCH 43/73] use referencing

---
 parser/Cargo.lock            |  16 ++-
 parser/Cargo.toml            |   1 +
 parser/src/json/normalize.rs | 206 +++++++++++++++++++++--------------
 rust/Cargo.lock              |  16 ++-
 sample_parser/Cargo.lock     |  16 ++-
 5 files changed, 173 insertions(+), 82 deletions(-)

diff --git a/parser/Cargo.lock b/parser/Cargo.lock
index adfc17c..2883a47 100644
--- a/parser/Cargo.lock
+++ b/parser/Cargo.lock
@@ -523,7 +523,7 @@ dependencies = [
  "num-cmp",
  "once_cell",
  "percent-encoding",
- "referencing",
+ "referencing 0.24.3",
  "regex-syntax",
  "serde",
  "serde_json",
@@ -566,6 +566,7 @@ dependencies = [
  "instant",
  "jsonschema",
  "lazy_static",
+ "referencing 0.26.1",
  "regex-syntax",
  "rustc-hash",
  "serde",
@@ -734,6 +735,19 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "referencing"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb853437e467c693ac1dc8c1520105a31b8c2588544ff2f3cfa5a7c706c6c069"
+dependencies = [
+ "ahash",
+ "fluent-uri",
+ "once_cell",
+ "percent-encoding",
+ "serde_json",
+]
+
 [[package]]
 name = "regex-automata"
 version = "0.4.8"
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index f37f3bf..93cf08e 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -16,6 +16,7 @@ url = "2.5.2"
 lazy_static = { version = "1.5.0", optional = true }
 regex-syntax = "0.8.5"
 indexmap = "2.6.0"
+referencing = "0.26.1"
 
 [features]
 default = ["jsonschema_validation", "lark"]
diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index 51fccc9..9aa2a91 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -1,7 +1,9 @@
 use anyhow::{anyhow, bail, Result};
 use indexmap::{IndexMap, IndexSet};
+use referencing::{Draft, Registry, Resolver, ResourceRef};
 use serde_json::{Map, Value};
 
+const DEFAULT_DRAFT: Draft = Draft::Draft202012;
 const TYPES: [&str; 6] = ["null", "boolean", "number", "string", "array", "object"];
 
 fn limited_str(node: &Value) -> String {
@@ -57,8 +59,7 @@ enum Schema {
     OneOf {
         options: Vec<Schema>,
     },
-    #[allow(dead_code)]
-    Ref(String),
+    Ref(Box<Schema>),
 }
 
 impl Schema {
@@ -177,16 +178,67 @@ impl Schema {
     }
 }
 
-impl TryFrom<Value> for Schema {
-    type Error = anyhow::Error;
+struct Context<'a> {
+    resolver: Resolver<'a>,
+    // TODO: actually use this to handle draft-specific behavior
+    draft: Draft,
+    // maybe... seen: Set<Schema>
+}
+
+impl<'a> Context<'a> {
+    fn in_subresource(&'a self, resource: ResourceRef) -> Result<Context<'a>> {
+        let resolver = self.resolver.in_subresource(resource)?;
+        Ok(Context {
+            resolver: resolver,
+            draft: resource.draft(),
+        })
+    }
+
+    fn as_resource_ref<'r>(&'a self, contents: &'r Value) -> ResourceRef<'r> {
+        self.draft
+            .detect(contents)
+            .unwrap_or(DEFAULT_DRAFT)
+            .create_resource_ref(contents)
+    }
 
-    fn try_from(value: Value) -> Result<Self, Self::Error> {
-        schema_from_value_inner(value)?.normalize()
+    fn lookup(&'a self, reference: &str) -> Result<ResourceRef> {
+        let resolved = self.resolver.lookup(reference)?;
+        Ok(self.as_resource_ref(&resolved.contents()))
     }
 }
 
-fn schema_from_value_inner(value: Value) -> Result<Schema> {
-    if let Some(b) = value.as_bool() {
+fn draft_for(value: &Value) -> Draft {
+    DEFAULT_DRAFT.detect(value).unwrap_or(DEFAULT_DRAFT)
+}
+
+fn build_schema(contents: &Value) -> Result<Schema> {
+    let draft = draft_for(contents);
+    let resource_ref = draft.create_resource_ref(contents);
+    let resource = draft.create_resource(contents.clone());
+    let base_uri = resource.id().unwrap_or("").to_string();
+
+    let registry = Registry::try_new(&base_uri, resource)?;
+
+    let resolver = registry.try_resolver(&base_uri)?;
+    let ctx = Context {
+        resolver: resolver,
+        draft: draft,
+    };
+
+    compile_resource(&ctx, resource_ref)?.normalize()
+}
+
+fn compile_resource(ctx: &Context, resource: ResourceRef) -> Result<Schema> {
+    let ctx = ctx.in_subresource(resource)?;
+    compile_contents(&ctx, resource.contents())
+}
+
+fn compile_contents(ctx: &Context, contents: &Value) -> Result<Schema> {
+    compile_contents_inner(ctx, contents)?.normalize()
+}
+
+fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
+    if let Some(b) = contents.as_bool() {
         return Ok({
             if b {
                 Schema::Any
@@ -200,7 +252,7 @@ fn schema_from_value_inner(value: Value) -> Result<Schema> {
 
     // Get the schema as an object
     // TODO: validate against metaschema & check for unimplemented keys
-    let schemadict = value
+    let schemadict = contents
         .as_object()
         .ok_or_else(|| anyhow!("schema must be an object or boolean"))?;
 
@@ -233,14 +285,14 @@ fn schema_from_value_inner(value: Value) -> Result<Schema> {
         let all_of = all_of
             .as_array()
             .ok_or_else(|| anyhow!("allOf must be an array"))?;
-        let siblings = Schema::try_from(Value::Object(schemadict))?;
+        let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
         // Short-circuit if schema is already unsatisfiable
         if matches!(siblings, Schema::Unsatisfiable { .. }) {
             return Ok(siblings);
         }
         let options = all_of
             .iter()
-            .map(|value| Schema::try_from(value.to_owned()))
+            .map(|value| compile_resource(&ctx, ctx.as_resource_ref(value)))
             .collect::<Result<Vec<_>>>()?;
         let merged = merge(options.iter().chain(vec![&siblings]).collect())?;
         return Ok(merged);
@@ -250,16 +302,14 @@ fn schema_from_value_inner(value: Value) -> Result<Schema> {
         let any_of = any_of
             .as_array()
             .ok_or_else(|| anyhow!("anyOf must be an array"))?;
-        let siblings = Schema::try_from(Value::Object(schemadict))?;
+        let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
         // Short-circuit if schema is already unsatisfiable
         if matches!(siblings, Schema::Unsatisfiable { .. }) {
             return Ok(siblings);
         }
         let options = any_of
             .iter()
-            .map(|value| {
-                Schema::try_from(value.to_owned()).and_then(|schema| merge_two(&schema, &siblings))
-            })
+            .map(|value| compile_resource(&ctx, ctx.as_resource_ref(value)))
             .collect::<Result<Vec<_>>>()?;
         return Ok(Schema::AnyOf { options });
     }
@@ -269,16 +319,14 @@ fn schema_from_value_inner(value: Value) -> Result<Schema> {
         let one_of = one_of
             .as_array()
             .ok_or_else(|| anyhow!("oneOf must be an array"))?;
-        let siblings = Schema::try_from(Value::Object(schemadict))?;
+        let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
         // Short-circuit if schema is already unsatisfiable
         if matches!(siblings, Schema::Unsatisfiable { .. }) {
             return Ok(siblings);
         }
         let options = one_of
             .iter()
-            .map(|value| {
-                Schema::try_from(value.to_owned()).and_then(|schema| merge_two(&schema, &siblings))
-            })
+            .map(|value| compile_resource(&ctx, ctx.as_resource_ref(value)))
             .collect::<Result<Vec<_>>>()?;
         return Ok(Schema::OneOf { options });
     }
@@ -288,17 +336,23 @@ fn schema_from_value_inner(value: Value) -> Result<Schema> {
             .as_str()
             .ok_or_else(|| anyhow!("$ref must be a string, got {}", limited_str(&reference)))?
             .to_string();
-        let siblings = Schema::try_from(Value::Object(schemadict))?;
+        let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
         // Short-circuit if schema is already unsatisfiable
         if matches!(siblings, Schema::Unsatisfiable { .. }) {
             return Ok(siblings);
         }
-        return Ok(merge_two(&siblings, &Schema::Ref(reference))?);
+        let resource = ctx.lookup(&reference)?;
+        let resolved_schema = compile_resource(ctx, resource)?;
+        if siblings != Schema::Any {
+            bail!("$ref with siblings not implemented")
+        }
+        // TODO: recursion...
+        return Ok(Schema::Ref(Box::new(resolved_schema)));
     }
 
     let types = match schemadict.remove("type") {
         Some(Value::String(tp)) => {
-            return try_type(&schemadict, &tp);
+            return compile_type(ctx, &tp, &schemadict);
         }
         Some(Value::Array(types)) => types
             .iter()
@@ -314,35 +368,37 @@ fn schema_from_value_inner(value: Value) -> Result<Schema> {
     // Shouldn't need siblings here since we've already handled allOf, anyOf, oneOf, and $ref
     let options = types
         .iter()
-        .map(|tp| try_type(&schemadict, &tp))
+        .map(|tp| compile_type(ctx, &tp, &schemadict))
         .collect::<Result<Vec<Schema>>>()?;
     Ok(Schema::AnyOf { options })
 }
 
-fn try_type(schema: &Map<String, Value>, tp: &str) -> Result<Schema> {
+fn compile_type(ctx: &Context, tp: &str, schema: &Map<String, Value>) -> Result<Schema> {
     match tp {
         "null" => Ok(Schema::Null),
         "boolean" => Ok(Schema::Boolean),
-        "number" | "integer" => try_numeric(
+        "number" | "integer" => compile_numeric(
             schema.get("minimum"),
             schema.get("maximum"),
             schema.get("exclusiveMinimum"),
             schema.get("exclusiveMaximum"),
             tp == "integer",
         ),
-        "string" => try_string(
+        "string" => compile_string(
             schema.get("minLength"),
             schema.get("maxLength"),
             schema.get("pattern"),
             schema.get("format"),
         ),
-        "array" => try_array(
+        "array" => compile_array(
+            ctx,
             schema.get("minItems"),
             schema.get("maxItems"),
             schema.get("prefixItems"),
             schema.get("items"),
         ),
-        "object" => try_object(
+        "object" => compile_object(
+            ctx,
             schema.get("properties"),
             schema.get("additionalProperties"),
             schema.get("required"),
@@ -351,7 +407,7 @@ fn try_type(schema: &Map<String, Value>, tp: &str) -> Result<Schema> {
     }
 }
 
-fn try_numeric(
+fn compile_numeric(
     minimum: Option<&Value>,
     maximum: Option<&Value>,
     exclusive_minimum: Option<&Value>,
@@ -405,7 +461,7 @@ fn try_numeric(
     })
 }
 
-fn try_string(
+fn compile_string(
     min_length: Option<&Value>,
     max_length: Option<&Value>,
     pattern: Option<&Value>,
@@ -448,7 +504,8 @@ fn try_string(
     })
 }
 
-fn try_array(
+fn compile_array(
+    ctx: &Context,
     min_items: Option<&Value>,
     max_items: Option<&Value>,
     prefix_items: Option<&Value>,
@@ -473,12 +530,12 @@ fn try_array(
             .as_array()
             .ok_or_else(|| anyhow!("Expected array for 'prefixItems', got {}", limited_str(val)))?
             .iter()
-            .map(|item| Schema::try_from(item.to_owned()))
+            .map(|item| compile_resource(&ctx, ctx.as_resource_ref(item)))
             .collect::<Result<Vec<Schema>>>()?,
     };
     let items = match items {
         None => None,
-        Some(val) => Some(Box::new(Schema::try_from(val.to_owned())?)),
+        Some(val) => Some(Box::new(compile_resource(&ctx, ctx.as_resource_ref(val))?)),
     };
     Ok(Schema::Array {
         min_items,
@@ -488,7 +545,8 @@ fn try_array(
     })
 }
 
-fn try_object(
+fn compile_object(
+    ctx: &Context,
     properties: Option<&Value>,
     additional_properties: Option<&Value>,
     required: Option<&Value>,
@@ -499,12 +557,12 @@ fn try_object(
             .as_object()
             .ok_or_else(|| anyhow!("Expected object for 'properties', got {}", limited_str(val)))?
             .iter()
-            .map(|(k, v)| Schema::try_from(v.to_owned()).map(|v| (k.clone(), v)))
+            .map(|(k, v)| compile_resource(&ctx, ctx.as_resource_ref(v)).map(|v| (k.clone(), v)))
             .collect::<Result<IndexMap<String, Schema>>>()?,
     };
     let additional_properties = match additional_properties {
         None => None,
-        Some(val) => Some(Box::new(Schema::try_from(val.to_owned())?)),
+        Some(val) => Some(Box::new(compile_resource(&ctx, ctx.as_resource_ref(val))?)),
     };
     let required = match required {
         None => IndexSet::new(),
@@ -563,12 +621,8 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
         (_, Schema::Any) => Ok(schema0.to_owned()),
         (Schema::Unsatisfiable { .. }, _) => Ok(schema0.to_owned()),
         (_, Schema::Unsatisfiable { .. }) => Ok(schema1.to_owned()),
-        (Schema::Ref(reference), _) => {
-            Err(anyhow!("$ref with siblings not implemented: {}", reference))
-        }
-        (_, Schema::Ref(reference)) => {
-            Err(anyhow!("$ref with siblings not implemented: {}", reference))
-        }
+        (Schema::Ref(_), _) => Err(anyhow!("$ref with siblings not implemented")),
+        (_, Schema::Ref(_)) => Err(anyhow!("$ref with siblings not implemented")),
         (Schema::OneOf { options }, _) => Ok(Schema::OneOf {
             options: options
                 .iter()
@@ -777,42 +831,6 @@ mod test {
     use super::*;
     use serde_json::json;
 
-    #[test]
-    fn test_normalize() {
-        let schema = json!({
-            "minLength": 1,
-            "maxLength": 10,
-            "pattern": "^[a-z]+$",
-            "format": "email",
-            "minimum": 10,
-        });
-        let schema = Schema::try_from(schema).unwrap();
-        println!("{:?}", schema);
-    }
-
-    #[test]
-    fn test_normalize_2() {
-        let schema = json!({
-            "anyOf": [
-                {
-                    "type": "string",
-                    "minLength": 1,
-                },
-                {
-                    "type": "number",
-                    "minimum": 10,
-                },
-            ],
-            "allOf" : [
-                {"type": "integer"}
-            ],
-            "pattern": "^[a-z]+$",
-            "maximum": 11
-        });
-        let schema = Schema::try_from(schema).unwrap();
-        println!("{:?}", schema);
-    }
-
     #[test]
     fn test_normalize_3() {
         let schema = json!({
@@ -838,7 +856,37 @@ mod test {
             ],
             "pattern": "^[a-z]+$",
         });
-        let schema = Schema::try_from(schema).unwrap();
+        let schema = build_schema(&schema).unwrap();
+        println!("{:?}", schema);
+    }
+
+    #[test]
+    fn test_ref() {
+        let schema = json!({
+            "type": "object",
+            "properties": {
+                "foo": {
+                    "$ref": "#ooo",
+                },
+                "bar": {
+                    "$id": "baz.com",
+                    "type": "array",
+                    "prefixItems": [
+                        {
+                            "$ref": "#/prefixItems/1"
+                        },
+                        {
+                            "type": "number",
+                        },
+                    ],
+                },
+                "baz": {
+                    "$anchor": "ooo",
+                    "type": "number",
+                },
+            },
+        });
+        let schema = build_schema(&schema).unwrap();
         println!("{:?}", schema);
     }
 }
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 11cc46c..2758cf2 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -529,7 +529,7 @@ dependencies = [
  "num-cmp",
  "once_cell",
  "percent-encoding",
- "referencing",
+ "referencing 0.24.3",
  "regex-syntax",
  "serde",
  "serde_json",
@@ -583,6 +583,7 @@ dependencies = [
  "instant",
  "jsonschema",
  "lazy_static",
+ "referencing 0.26.1",
  "regex-syntax",
  "rustc-hash",
  "serde",
@@ -871,6 +872,19 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "referencing"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb853437e467c693ac1dc8c1520105a31b8c2588544ff2f3cfa5a7c706c6c069"
+dependencies = [
+ "ahash",
+ "fluent-uri",
+ "once_cell",
+ "percent-encoding",
+ "serde_json",
+]
+
 [[package]]
 name = "regex-automata"
 version = "0.4.8"
diff --git a/sample_parser/Cargo.lock b/sample_parser/Cargo.lock
index 2d41c27..09f9e8d 100644
--- a/sample_parser/Cargo.lock
+++ b/sample_parser/Cargo.lock
@@ -812,7 +812,7 @@ dependencies = [
  "num-cmp",
  "once_cell",
  "percent-encoding",
- "referencing",
+ "referencing 0.24.3",
  "regex-syntax",
  "serde",
  "serde_json",
@@ -865,6 +865,7 @@ dependencies = [
  "instant",
  "jsonschema",
  "lazy_static",
+ "referencing 0.26.1",
  "regex-syntax",
  "rustc-hash",
  "serde",
@@ -1289,6 +1290,19 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "referencing"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb853437e467c693ac1dc8c1520105a31b8c2588544ff2f3cfa5a7c706c6c069"
+dependencies = [
+ "ahash",
+ "fluent-uri",
+ "once_cell",
+ "percent-encoding",
+ "serde_json",
+]
+
 [[package]]
 name = "regex"
 version = "1.11.1"

From fa080a812266116a2bbf2be7f627b5e38ef93626 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Thu, 14 Nov 2024 22:00:28 -0800
Subject: [PATCH 44/73] abspath

---
 parser/src/json/normalize.rs | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/parser/src/json/normalize.rs b/parser/src/json/normalize.rs
index 9aa2a91..0703739 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/normalize.rs
@@ -1,5 +1,6 @@
 use anyhow::{anyhow, bail, Result};
 use indexmap::{IndexMap, IndexSet};
+use jsonschema::Uri;
 use referencing::{Draft, Registry, Resolver, ResourceRef};
 use serde_json::{Map, Value};
 
@@ -59,7 +60,10 @@ enum Schema {
     OneOf {
         options: Vec<Schema>,
     },
-    Ref(Box<Schema>),
+    Ref {
+        uri: String,
+        schema: Box<Schema>,
+    },
 }
 
 impl Schema {
@@ -173,7 +177,7 @@ impl Schema {
                 Ok(Schema::OneOf { options: valid })
             }
             // TODO: ?
-            Schema::Ref(..) => Ok(self),
+            Schema::Ref{ .. } => Ok(self),
         }
     }
 }
@@ -201,6 +205,10 @@ impl<'a> Context<'a> {
             .create_resource_ref(contents)
     }
 
+    fn get_abspath(&self, reference: &str) -> Result<Uri<String>> {
+        Ok(self.resolver.resolve_against(&self.resolver.base_uri().borrow(), reference)?.normalize())
+    }
+
     fn lookup(&'a self, reference: &str) -> Result<ResourceRef> {
         let resolved = self.resolver.lookup(reference)?;
         Ok(self.as_resource_ref(&resolved.contents()))
@@ -225,7 +233,7 @@ fn build_schema(contents: &Value) -> Result<Schema> {
         draft: draft,
     };
 
-    compile_resource(&ctx, resource_ref)?.normalize()
+    compile_resource(&ctx, resource_ref)
 }
 
 fn compile_resource(ctx: &Context, resource: ResourceRef) -> Result<Schema> {
@@ -341,13 +349,14 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
         if matches!(siblings, Schema::Unsatisfiable { .. }) {
             return Ok(siblings);
         }
+        let uri = ctx.get_abspath(&reference)?;
         let resource = ctx.lookup(&reference)?;
         let resolved_schema = compile_resource(ctx, resource)?;
         if siblings != Schema::Any {
             bail!("$ref with siblings not implemented")
         }
         // TODO: recursion...
-        return Ok(Schema::Ref(Box::new(resolved_schema)));
+        return Ok(Schema::Ref { uri: uri.into_string(), schema: Box::new(resolved_schema) });
     }
 
     let types = match schemadict.remove("type") {
@@ -621,8 +630,8 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
         (_, Schema::Any) => Ok(schema0.to_owned()),
         (Schema::Unsatisfiable { .. }, _) => Ok(schema0.to_owned()),
         (_, Schema::Unsatisfiable { .. }) => Ok(schema1.to_owned()),
-        (Schema::Ref(_), _) => Err(anyhow!("$ref with siblings not implemented")),
-        (_, Schema::Ref(_)) => Err(anyhow!("$ref with siblings not implemented")),
+        (Schema::Ref { .. }, _) => Err(anyhow!("$ref with siblings not implemented")),
+        (_, Schema::Ref { .. }) => Err(anyhow!("$ref with siblings not implemented")),
         (Schema::OneOf { options }, _) => Ok(Schema::OneOf {
             options: options
                 .iter()

From b7af4f3726d288192d471b8757c6a69e67e25f4f Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 15 Nov 2024 11:06:31 -0800
Subject: [PATCH 45/73] attempt at recursive refs

---
 parser/src/json/mod.rs                      |  2 +-
 parser/src/json/{normalize.rs => schema.rs} | 95 +++++++++++++++++----
 2 files changed, 80 insertions(+), 17 deletions(-)
 rename parser/src/json/{normalize.rs => schema.rs} (90%)

diff --git a/parser/src/json/mod.rs b/parser/src/json/mod.rs
index 0509f4e..49e7237 100644
--- a/parser/src/json/mod.rs
+++ b/parser/src/json/mod.rs
@@ -1,4 +1,4 @@
 pub mod compiler;
 mod formats;
-mod normalize;
+mod schema;
 mod numeric;
diff --git a/parser/src/json/normalize.rs b/parser/src/json/schema.rs
similarity index 90%
rename from parser/src/json/normalize.rs
rename to parser/src/json/schema.rs
index 0703739..fcf0168 100644
--- a/parser/src/json/normalize.rs
+++ b/parser/src/json/schema.rs
@@ -1,3 +1,5 @@
+use std::{cell::RefCell, collections::HashMap, rc::Rc};
+
 use anyhow::{anyhow, bail, Result};
 use indexmap::{IndexMap, IndexSet};
 use jsonschema::Uri;
@@ -17,7 +19,7 @@ fn limited_str(node: &Value) -> String {
 }
 
 #[derive(Debug, PartialEq, Clone)]
-enum Schema {
+pub enum Schema {
     Any,
     Unsatisfiable {
         reason: String,
@@ -62,7 +64,7 @@ enum Schema {
     },
     Ref {
         uri: String,
-        schema: Box<Schema>,
+        schema: Option<Box<Schema>>
     },
 }
 
@@ -186,7 +188,7 @@ struct Context<'a> {
     resolver: Resolver<'a>,
     // TODO: actually use this to handle draft-specific behavior
     draft: Draft,
-    // maybe... seen: Set<Schema>
+    refs: Rc<RefCell<HashMap<String, Rc<RefCell<Schema>>>>>,
 }
 
 impl<'a> Context<'a> {
@@ -195,6 +197,7 @@ impl<'a> Context<'a> {
         Ok(Context {
             resolver: resolver,
             draft: resource.draft(),
+            refs: Rc::clone(&self.refs),
         })
     }
 
@@ -205,21 +208,33 @@ impl<'a> Context<'a> {
             .create_resource_ref(contents)
     }
 
-    fn get_abspath(&self, reference: &str) -> Result<Uri<String>> {
+    fn normalize_ref(&self, reference: &str) -> Result<Uri<String>> {
         Ok(self.resolver.resolve_against(&self.resolver.base_uri().borrow(), reference)?.normalize())
     }
 
-    fn lookup(&'a self, reference: &str) -> Result<ResourceRef> {
+    fn lookup_resource(&'a self, reference: &str) -> Result<ResourceRef> {
         let resolved = self.resolver.lookup(reference)?;
         Ok(self.as_resource_ref(&resolved.contents()))
     }
+
+    fn insert_ref(&self, uri: &str, schema: Schema) {
+        self.refs.borrow_mut().insert(uri.to_string(), Rc::new(RefCell::new(schema)));
+    }
+
+    fn get_ref(&self, uri: &str) -> Option<Schema> {
+        self.refs.borrow().get(uri).map(|schema| schema.borrow().clone())
+    }
+
+    fn get_ref_mut(&self, uri: &str) -> Option<Rc<RefCell<Schema>>> {
+        self.refs.borrow().get(uri).cloned()
+    }
 }
 
 fn draft_for(value: &Value) -> Draft {
     DEFAULT_DRAFT.detect(value).unwrap_or(DEFAULT_DRAFT)
 }
 
-fn build_schema(contents: &Value) -> Result<Schema> {
+pub fn build_schema(contents: &Value) -> Result<Schema> {
     let draft = draft_for(contents);
     let resource_ref = draft.create_resource_ref(contents);
     let resource = draft.create_resource(contents.clone());
@@ -231,6 +246,7 @@ fn build_schema(contents: &Value) -> Result<Schema> {
     let ctx = Context {
         resolver: resolver,
         draft: draft,
+        refs: Rc::new(RefCell::new(HashMap::new())),
     };
 
     compile_resource(&ctx, resource_ref)
@@ -344,19 +360,47 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
             .as_str()
             .ok_or_else(|| anyhow!("$ref must be a string, got {}", limited_str(&reference)))?
             .to_string();
+
         let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
-        // Short-circuit if schema is already unsatisfiable
-        if matches!(siblings, Schema::Unsatisfiable { .. }) {
-            return Ok(siblings);
-        }
-        let uri = ctx.get_abspath(&reference)?;
-        let resource = ctx.lookup(&reference)?;
-        let resolved_schema = compile_resource(ctx, resource)?;
-        if siblings != Schema::Any {
+        if siblings == Schema::Any {
+            let uri = ctx.normalize_ref(&reference)?.into_string();
+            if let Some(schema) = ctx.get_ref(&uri) {
+                return Ok(schema);
+            }
+            let placeholder = Schema::Ref {
+                uri: uri.clone(),
+                schema: None,
+            };
+            ctx.insert_ref(uri.as_str(), placeholder);
+            let resource = ctx.lookup_resource(&reference)?;
+            let resolved_schema = compile_resource(ctx, resource)?;
+            let mut placeholder = ctx.get_ref_mut(&uri);
+            if let Some(placeholder) = placeholder.as_mut() {
+                if matches!(*placeholder.borrow(), Schema::Ref { .. }) {
+                    *placeholder.borrow_mut() = Schema::Ref {
+                        uri: uri.clone(),
+                        schema: Some(Box::new(resolved_schema)),
+                    };
+                }
+                return Ok(placeholder.borrow().clone());
+            }
+            bail!("$ref placeholder not found");
+        } else {
             bail!("$ref with siblings not implemented")
         }
-        // TODO: recursion...
-        return Ok(Schema::Ref { uri: uri.into_string(), schema: Box::new(resolved_schema) });
+
+        // // Short-circuit if schema is already unsatisfiable
+        // if matches!(siblings, Schema::Unsatisfiable { .. }) {
+        //     return Ok(siblings);
+        // }
+        // let uri = ctx.normalize_ref(&reference)?;
+        // let resource = ctx.lookup_resource(&reference)?;
+        // let resolved_schema = compile_resource(ctx, resource)?;
+        // if siblings != Schema::Any {
+        //     bail!("$ref with siblings not implemented")
+        // }
+        // // TODO: recursion...
+        // return Ok(Schema::Ref { uri: uri.into_string(), schema: Box::new(resolved_schema) });
     }
 
     let types = match schemadict.remove("type") {
@@ -840,6 +884,25 @@ mod test {
     use super::*;
     use serde_json::json;
 
+    #[test]
+    fn test_linked_list_simple() {
+        let schema = json!({
+            "type": "object",
+            "properties": {
+                "value": {
+                    "type": "number",
+                },
+                "next": {
+                    "$ref": "#",
+                },
+            },
+            "additionalProperties": false,
+            "required": ["value"],
+        });
+        let schema = build_schema(&schema).unwrap();
+        println!("{:?}", schema);
+    }
+
     #[test]
     fn test_normalize_3() {
         let schema = json!({

From 063d0a195146445d12b34f7f4cd53301204d69ab Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 15 Nov 2024 11:48:08 -0800
Subject: [PATCH 46/73] simplify refs

---
 parser/src/json/schema.rs | 61 ++++++++++++---------------------------
 1 file changed, 18 insertions(+), 43 deletions(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index fcf0168..6077579 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -64,7 +64,6 @@ pub enum Schema {
     },
     Ref {
         uri: String,
-        schema: Option<Box<Schema>>
     },
 }
 
@@ -188,7 +187,7 @@ struct Context<'a> {
     resolver: Resolver<'a>,
     // TODO: actually use this to handle draft-specific behavior
     draft: Draft,
-    refs: Rc<RefCell<HashMap<String, Rc<RefCell<Schema>>>>>,
+    defs: Rc<RefCell<HashMap<String, Schema>>>,
 }
 
 impl<'a> Context<'a> {
@@ -197,7 +196,7 @@ impl<'a> Context<'a> {
         Ok(Context {
             resolver: resolver,
             draft: resource.draft(),
-            refs: Rc::clone(&self.refs),
+            defs: Rc::clone(&self.defs),
         })
     }
 
@@ -218,15 +217,11 @@ impl<'a> Context<'a> {
     }
 
     fn insert_ref(&self, uri: &str, schema: Schema) {
-        self.refs.borrow_mut().insert(uri.to_string(), Rc::new(RefCell::new(schema)));
+        self.defs.borrow_mut().insert(uri.to_string(), schema);
     }
 
-    fn get_ref(&self, uri: &str) -> Option<Schema> {
-        self.refs.borrow().get(uri).map(|schema| schema.borrow().clone())
-    }
-
-    fn get_ref_mut(&self, uri: &str) -> Option<Rc<RefCell<Schema>>> {
-        self.refs.borrow().get(uri).cloned()
+    fn contains_ref(&self, uri: &str) -> bool {
+        self.defs.borrow().contains_key(uri)
     }
 }
 
@@ -234,7 +229,7 @@ fn draft_for(value: &Value) -> Draft {
     DEFAULT_DRAFT.detect(value).unwrap_or(DEFAULT_DRAFT)
 }
 
-pub fn build_schema(contents: &Value) -> Result<Schema> {
+pub fn build_schema(contents: &Value) -> Result<(Schema, HashMap<String, Schema>)> {
     let draft = draft_for(contents);
     let resource_ref = draft.create_resource_ref(contents);
     let resource = draft.create_resource(contents.clone());
@@ -246,10 +241,12 @@ pub fn build_schema(contents: &Value) -> Result<Schema> {
     let ctx = Context {
         resolver: resolver,
         draft: draft,
-        refs: Rc::new(RefCell::new(HashMap::new())),
+        defs: Rc::new(RefCell::new(HashMap::new())),
     };
 
-    compile_resource(&ctx, resource_ref)
+    let schema = compile_resource(&ctx, resource_ref)?;
+    let definitions = ctx.defs.take();
+    Ok((schema, definitions))
 }
 
 fn compile_resource(ctx: &Context, resource: ResourceRef) -> Result<Schema> {
@@ -364,43 +361,21 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
         let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
         if siblings == Schema::Any {
             let uri = ctx.normalize_ref(&reference)?.into_string();
-            if let Some(schema) = ctx.get_ref(&uri) {
-                return Ok(schema);
-            }
             let placeholder = Schema::Ref {
-                uri: uri.clone(),
-                schema: None,
+                uri: uri.clone()
             };
-            ctx.insert_ref(uri.as_str(), placeholder);
+            if ctx.contains_ref(uri.as_str()) {
+                // We've already built this ref, so we can just return it
+                return Ok(placeholder);
+            }
+            ctx.insert_ref(uri.as_str(), placeholder.clone());
             let resource = ctx.lookup_resource(&reference)?;
             let resolved_schema = compile_resource(ctx, resource)?;
-            let mut placeholder = ctx.get_ref_mut(&uri);
-            if let Some(placeholder) = placeholder.as_mut() {
-                if matches!(*placeholder.borrow(), Schema::Ref { .. }) {
-                    *placeholder.borrow_mut() = Schema::Ref {
-                        uri: uri.clone(),
-                        schema: Some(Box::new(resolved_schema)),
-                    };
-                }
-                return Ok(placeholder.borrow().clone());
-            }
-            bail!("$ref placeholder not found");
+            ctx.insert_ref(uri.as_str(), resolved_schema.clone());
+            return Ok(placeholder);
         } else {
             bail!("$ref with siblings not implemented")
         }
-
-        // // Short-circuit if schema is already unsatisfiable
-        // if matches!(siblings, Schema::Unsatisfiable { .. }) {
-        //     return Ok(siblings);
-        // }
-        // let uri = ctx.normalize_ref(&reference)?;
-        // let resource = ctx.lookup_resource(&reference)?;
-        // let resolved_schema = compile_resource(ctx, resource)?;
-        // if siblings != Schema::Any {
-        //     bail!("$ref with siblings not implemented")
-        // }
-        // // TODO: recursion...
-        // return Ok(Schema::Ref { uri: uri.into_string(), schema: Box::new(resolved_schema) });
     }
 
     let types = match schemadict.remove("type") {

From c8bc68ac96e40baf43e54edeb27b85f7969abb07 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 15 Nov 2024 12:19:53 -0800
Subject: [PATCH 47/73] explicit default root uri

---
 parser/src/json/schema.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index 6077579..0db7ffe 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -6,6 +6,7 @@ use jsonschema::Uri;
 use referencing::{Draft, Registry, Resolver, ResourceRef};
 use serde_json::{Map, Value};
 
+const DEFAULT_ROOT_URI: &str = "json-schema:///";
 const DEFAULT_DRAFT: Draft = Draft::Draft202012;
 const TYPES: [&str; 6] = ["null", "boolean", "number", "string", "array", "object"];
 
@@ -233,7 +234,7 @@ pub fn build_schema(contents: &Value) -> Result<(Schema, HashMap<String, Schema>
     let draft = draft_for(contents);
     let resource_ref = draft.create_resource_ref(contents);
     let resource = draft.create_resource(contents.clone());
-    let base_uri = resource.id().unwrap_or("").to_string();
+    let base_uri = resource.id().unwrap_or(DEFAULT_ROOT_URI).to_string();
 
     let registry = Registry::try_new(&base_uri, resource)?;
 

From e0f583260923f9fa411c7c18ebc752e90e52085f Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 15 Nov 2024 12:31:04 -0800
Subject: [PATCH 48/73] comment

---
 parser/src/json/schema.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index 0db7ffe..de6e472 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -457,6 +457,7 @@ fn compile_numeric(
                 .ok_or_else(|| anyhow!("Expected f64 for 'maximum', got {}", limited_str(val)))?,
         ),
     };
+    // TODO: actually use ctx.draft to determine which style of exclusiveMinimum/Maximum to use
     let exclusive_minimum = match exclusive_minimum {
         // Draft4-style boolean values
         None | Some(Value::Bool(false)) => None,

From 87203f9836d106fe30efa8a2ac8201ffb6f15bf6 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 15 Nov 2024 12:45:16 -0800
Subject: [PATCH 49/73] allow sibling keys on refs as long as they aren't too
 recursive

---
 parser/src/json/schema.rs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index de6e472..a2789be 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -359,9 +359,9 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
             .ok_or_else(|| anyhow!("$ref must be a string, got {}", limited_str(&reference)))?
             .to_string();
 
+        let uri: String = ctx.normalize_ref(&reference)?.into_string();
         let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
         if siblings == Schema::Any {
-            let uri = ctx.normalize_ref(&reference)?.into_string();
             let placeholder = Schema::Ref {
                 uri: uri.clone()
             };
@@ -375,7 +375,12 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
             ctx.insert_ref(uri.as_str(), resolved_schema.clone());
             return Ok(placeholder);
         } else {
-            bail!("$ref with siblings not implemented")
+            if ctx.contains_ref(uri.as_str()) {
+                bail!("Recursive $refs with sibling keys not implemented")
+            }
+            let resource = ctx.lookup_resource(&reference)?;
+            let resolved_schema = compile_resource(ctx, resource)?;
+            return Ok(merge_two(&siblings, &resolved_schema)?);
         }
     }
 

From 6f0d336f751f29eab535c77d3e21b7c1f9556aa5 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 15 Nov 2024 13:12:30 -0800
Subject: [PATCH 50/73] from bool

---
 parser/src/json/schema.rs | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index a2789be..89a3eab 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -68,6 +68,18 @@ pub enum Schema {
     },
 }
 
+impl From<bool> for Schema {
+    fn from(b: bool) -> Schema {
+        if b {
+            Schema::Any
+        } else {
+            Schema::Unsatisfiable {
+                reason: "Schema is false".to_string(),
+            }
+        }
+    }
+}
+
 impl Schema {
     fn normalize(self) -> Result<Schema> {
         match self {
@@ -231,6 +243,10 @@ fn draft_for(value: &Value) -> Draft {
 }
 
 pub fn build_schema(contents: &Value) -> Result<(Schema, HashMap<String, Schema>)> {
+    if let Some(b) = contents.as_bool() {
+        return Ok((b.into(), HashMap::new()));
+    }
+
     let draft = draft_for(contents);
     let resource_ref = draft.create_resource_ref(contents);
     let resource = draft.create_resource(contents.clone());
@@ -261,15 +277,7 @@ fn compile_contents(ctx: &Context, contents: &Value) -> Result<Schema> {
 
 fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
     if let Some(b) = contents.as_bool() {
-        return Ok({
-            if b {
-                Schema::Any
-            } else {
-                Schema::Unsatisfiable {
-                    reason: "schema is false".to_string(),
-                }
-            }
-        });
+        return Ok(b.into());
     }
 
     // Get the schema as an object

From 32b1cc09ef242373a7c8e63c0b0479f49cfcc864 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 15 Nov 2024 16:49:58 -0800
Subject: [PATCH 51/73] use schema.rs

---
 parser/src/json/compiler.rs | 496 ++++++++++--------------------------
 1 file changed, 136 insertions(+), 360 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index 2bfdf2e..3f866a0 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -1,7 +1,9 @@
 use anyhow::{anyhow, bail, Result};
+use indexmap::IndexMap;
 use serde_json::{json, Value};
 use std::{collections::HashMap, vec};
 
+use super::schema::{build_schema, Schema};
 use super::formats::lookup_format;
 use super::numeric::{rx_float_range, rx_int_range};
 use crate::{
@@ -79,15 +81,6 @@ const NUMERIC_KEYS: [&str; 4] = ["minimum", "maximum", "exclusiveMinimum", "excl
 
 const CHAR_REGEX: &str = r#"(\\([\"\\\/bfnrt]|u[a-fA-F0-9]{4})|[^\"\\\x00-\x1F\x7F])"#;
 
-fn limited_str(node: &Value) -> String {
-    let s = node.to_string();
-    if s.len() > 100 {
-        format!("{}...", &s[..100])
-    } else {
-        s
-    }
-}
-
 fn validate_json_node_keys(node: &Value) -> Result<()> {
     let node = node
         .as_object()
@@ -141,7 +134,7 @@ macro_rules! cache {
         if $field.is_none() {
             $field = Some($gen);
         }
-        return $field.unwrap();
+        return Ok($field.unwrap());
     };
 }
 
@@ -179,87 +172,6 @@ fn mk_regex(rx: &str) -> RegexSpec {
     RegexSpec::Regex(rx.to_string())
 }
 
-trait OptionalField {
-    fn opt_u64(&self, key: &str) -> Result<Option<u64>>;
-    fn opt_f64(&self, key: &str) -> Result<Option<f64>>;
-    fn opt_str(&self, key: &str) -> Result<Option<&str>>;
-    fn opt_array(&self, key: &str) -> Result<Option<&Vec<Value>>>;
-    #[allow(dead_code)]
-    fn opt_bool(&self, key: &str) -> Result<Option<bool>>;
-    fn opt_object(&self, key: &str) -> Result<Option<&serde_json::Map<String, Value>>>;
-}
-
-fn expected_err(key: &str, val: &Value, expected: &str) -> anyhow::Error {
-    anyhow!(
-        "Expected {} for field {:?}, got: {}",
-        expected,
-        key,
-        limited_str(val)
-    )
-}
-
-impl OptionalField for Value {
-    fn opt_u64(&self, key: &str) -> Result<Option<u64>> {
-        if let Some(val) = self.get(key) {
-            val.as_u64()
-                .ok_or_else(|| expected_err(key, val, "unsigned integer"))
-                .map(Some)
-        } else {
-            Ok(None)
-        }
-    }
-
-    fn opt_f64(&self, key: &str) -> Result<Option<f64>> {
-        if let Some(val) = self.get(key) {
-            val.as_f64()
-                .ok_or_else(|| expected_err(key, val, "float"))
-                .map(Some)
-        } else {
-            Ok(None)
-        }
-    }
-
-    fn opt_str(&self, key: &str) -> Result<Option<&str>> {
-        if let Some(val) = self.get(key) {
-            val.as_str()
-                .ok_or_else(|| expected_err(key, val, "string"))
-                .map(Some)
-        } else {
-            Ok(None)
-        }
-    }
-
-    fn opt_array(&self, key: &str) -> Result<Option<&Vec<Value>>> {
-        if let Some(val) = self.get(key) {
-            val.as_array()
-                .ok_or_else(|| expected_err(key, val, "array"))
-                .map(Some)
-        } else {
-            Ok(None)
-        }
-    }
-
-    fn opt_bool(&self, key: &str) -> Result<Option<bool>> {
-        if let Some(val) = self.get(key) {
-            val.as_bool()
-                .ok_or_else(|| expected_err(key, val, "boolean"))
-                .map(Some)
-        } else {
-            Ok(None)
-        }
-    }
-
-    fn opt_object(&self, key: &str) -> Result<Option<&serde_json::Map<String, Value>>> {
-        if let Some(val) = self.get(key) {
-            val.as_object()
-                .ok_or_else(|| expected_err(key, val, "object"))
-                .map(Some)
-        } else {
-            Ok(None)
-        }
-    }
-}
-
 impl Compiler {
     pub fn new(options: JsonCompileOptions) -> Self {
         Self {
@@ -282,7 +194,9 @@ impl Compiler {
             ..GrammarWithLexer::default()
         });
 
-        let root = self.gen_json(schema)?;
+        let (compiled_schema, definitions) = build_schema(schema)?;
+
+        let root = self.gen_json(&compiled_schema)?;
         self.builder.set_start_node(root);
 
         while let Some((path0, pl)) = self.pending_definitions.pop() {
@@ -296,243 +210,119 @@ impl Compiler {
             if node.is_null() {
                 bail!("Definition not found: {}", path0);
             }
-
-            let compiled = self.gen_json(node)?;
+            let schema = definitions.get(&path0).unwrap();
+            let compiled = self.gen_json(schema)?;
             self.builder.set_placeholder(pl, compiled);
         }
 
         Ok(())
     }
 
-    fn process_any_of(&mut self, obj: &Value) -> Result<NodeRef> {
-        let arr = obj.as_array().ok_or_else(|| anyhow!("Expected array in anyOf, got: {}", limited_str(obj)))?;
-        if arr.is_empty() {
-            return Err(anyhow!(UnsatisfiableSchemaError {
-                message: "anyOf has no schemas".to_string(),
-            }));
-        }
-        // Unwrap, filtering out unsatisfiable schemas
-        let options = arr.iter()
-            .map(|json_schema| self.gen_json(json_schema))
-            .filter_map(|res| {
-                match res {
-                    Ok(node) => Some(Ok(node)),
-                    Err(e) => match e.downcast_ref::<UnsatisfiableSchemaError>() {
-                        // If it's not an UnsatisfiableSchemaError, just propagate it normally
-                        None => { Some(Err(e)) },
-                        // Ignore unsatisfiable schemas
-                        Some(_) => None,
+    fn gen_json(&mut self, json_schema: &Schema) -> Result<NodeRef> {
+        match json_schema {
+            Schema::Any => self.gen_json_any(),
+            Schema::Unsatisfiable { reason } => Err(anyhow!(UnsatisfiableSchemaError {
+                message: reason.to_string(),
+            })),
+            Schema::Null => Ok(self.builder.string("null")),
+            Schema::Boolean => Ok(self.lexeme(r"true|false")),
+            Schema::Number {
+                minimum,
+                maximum,
+                exclusive_minimum,
+                exclusive_maximum ,
+                integer
+            } => {
+                let (minimum, exclusive_minimum) = match (minimum, exclusive_minimum) {
+                    (Some(min), Some(xmin)) => {
+                        if xmin >= min {
+                            (Some(*xmin), true)
+                        } else {
+                            (Some(*min), false)
+                        }
                     }
+                    (Some(min), None) => (Some(*min), false),
+                    (None, Some(xmin)) => (Some(*xmin), true),
+                    (None, None) => (None, false),
+                };
+                let (maximum, exclusive_maximum) = match (maximum, exclusive_maximum) {
+                    (Some(max), Some(xmax)) => {
+                        if xmax <= max {
+                            (Some(*xmax), true)
+                        } else {
+                            (Some(*max), false)
+                        }
+                    }
+                    (Some(max), None) => (Some(*max), false),
+                    (None, Some(xmax)) => (Some(*xmax), true),
+                    (None, None) => (None, false),
+                };
+                if *integer {
+                    self.json_int(minimum, maximum, exclusive_minimum, exclusive_maximum)
+                } else {
+                    self.json_number(minimum, maximum, exclusive_minimum, exclusive_maximum)
                 }
-            })
-            .collect::<Result<Vec<NodeRef>>>()?;
-        if options.is_empty() {
-            // If all schemas are unsatisfiable, return an error
-            // We can't really point to any specific schema at fault, so just dump the whole anyOf
-            return Err(anyhow!(UnsatisfiableSchemaError {
-                message: format!("all schemas in anyOf are unsatisfiable: {}", json_dumps(obj)),
-            }));
+            },
+            Schema::String {
+                min_length,
+                max_length,
+                pattern,
+                format,
+            } => self.gen_json_string(*min_length, *max_length, pattern.as_deref(), format.as_deref()),
+            Schema::Array {
+                min_items,
+                max_items,
+                prefix_items,
+                items,
+            } => self.gen_json_array(prefix_items, items.as_deref().unwrap_or(&true.into()), *min_items, *max_items),
+            Schema::Object {
+                properties,
+                additional_properties,
+                required,
+            } => self.gen_json_object(properties, additional_properties.as_deref().unwrap_or(&true.into()), required.iter().cloned().collect()),
+            Schema::Const { value } => self.gen_json_const(value.clone()),
+            Schema::Enum { options } => self.gen_json_enum(options.clone()),
+            Schema::AnyOf { options } => self.process_any_of(options.clone()),
+            Schema::OneOf { options } => self.process_any_of(options.clone()),
+            Schema::Ref { uri, .. } => self.get_definition(uri),
         }
-        Ok(self.builder.select(&options))
     }
 
-    fn gen_json(&mut self, json_schema: &Value) -> Result<NodeRef> {
-        if json_schema.as_bool() == Some(true) {
-            return Ok(self.gen_json_any());
-        }
-
-        if json_schema.as_bool() == Some(false) {
-            return Err(anyhow!(UnsatisfiableSchemaError {
-                message: "false schema".to_string(),
-            }));
-        }
-
-        if let Some(json_schema) = json_schema.as_object() {
-            // TODO: should be sufficient to have only ignored keys here
-            if json_schema.is_empty() {
-                return Ok(self.gen_json_any());
-            }
-        }
-
-        // eprintln!("gen_json: {}", limited_str(json_schema));
-        validate_json_node_keys(json_schema)?;
-
-        // Process anyOf
-        if let Some(any_of) = json_schema.get("anyOf") {
-            return self.process_any_of(any_of);
-        }
-
-        // Process oneOf (same handling as anyOf for now)
-        if let Some(one_of) = json_schema.get("oneOf") {
-            return self.process_any_of(one_of);
-        }
-
-        // Process allOf
-        if let Some(all_of_list) = json_schema.opt_array("allOf")? {
-            if all_of_list.len() != 1 {
-                bail!("Only support allOf with exactly one item");
-            }
-            return self.gen_json(&all_of_list[0]);
-        }
-
-        // Process $ref
-        if let Some(reference) = json_schema.get("$ref") {
-            let ref_str = reference.as_str().ok_or_else(|| {
-                anyhow!("Expected string in $ref, got: {}", limited_str(reference))
-            })?;
-            return self.get_definition(ref_str);
-        }
-
-        // Process const
-        if let Some(const_value) = json_schema.get("const") {
-            return self.gen_json_const(const_value);
-        }
-
-        // Process enum
-        if let Some(enum_array) = json_schema.opt_array("enum")? {
-            let options = enum_array
-                .iter()
-                .map(|opt| self.gen_json_const(opt))
-                .collect::<Result<Vec<_>, _>>()?;
-            return Ok(self.builder.select(&options));
-        }
-
-        // Process type
-        if let Some(tp) = json_schema.opt_str("type")? {
-            return self.gen_json_type(tp, json_schema)
-        }
-
-        let tps = TYPES.iter().map(|s| Value::String(s.to_string())).collect::<Vec<_>>();
-        let types = match json_schema.opt_array("type")? {
-            Some(types) => types,
-            None => &tps,
-        };
+    fn process_any_of(&mut self, options: Vec<Schema>) -> Result<NodeRef> {
+        let options = options.iter()
+            .map(|v| self.gen_json(v))
+            .collect::<Result<Vec<_>>>()?;
+        Ok(self.builder.select(&options))
+    }
 
-        let nodes = types
-            .iter()
-            .map(|v| {
-                let tp = v.as_str().ok_or_else(|| {
-                    anyhow!("Expected string in type list, got: {}", limited_str(v))
-                })?;
-                self.gen_json_type(tp, json_schema)
-            })
+    fn gen_json_enum(&mut self, options: Vec<Value>) -> Result<NodeRef> {
+        let options = options.into_iter()
+            .map(|v| self.gen_json_const(v))
             .collect::<Result<Vec<_>>>()?;
-        Ok(self.builder.select(&nodes))
+        Ok(self.builder.select(&options))
     }
 
-    fn gen_json_const(&mut self, const_value: &Value) -> Result<NodeRef> {
+    fn gen_json_const(&mut self, const_value: Value) -> Result<NodeRef> {
         // Recursively build a grammar for a constant value (just to play nicely with separators and whitespace flexibility)
         match const_value {
             Value::Object(values) => {
-                let properties = values.iter()
-                    .map(|(k, v)| {
-                        (k.clone(), json!({"const": v}))
-                    })
-                    .collect::<serde_json::Map<_, _>>();
-                self.gen_json_object(&properties, &Value::Bool(false), properties.keys().cloned().collect())
+                let properties = IndexMap::from_iter(values.into_iter().map(|(k, v)| (k, Schema::Const { value: v })));
+                let required = properties.keys().cloned().collect();
+                self.gen_json_object(&properties, &false.into(), required)
             },
             Value::Array(values) => {
                 let n_items = values.len() as u64;
-                let prefix_items = values.iter().map(|v| json!({"const": v})).collect::<Vec<_>>();
-                self.gen_json_array(&prefix_items, &Value::Bool(false), n_items, Some(n_items))
+                let prefix_items = values.into_iter().map(|v| Schema::Const { value: v }).collect::<Vec<_>>();
+                self.gen_json_array(&prefix_items, &false.into(), n_items, Some(n_items))
             },
             _ => {
                 // let serde_json dump simple values
-                let const_str = json_dumps(const_value);
+                let const_str = json_dumps(&const_value);
                 Ok(self.builder.string(&const_str))
             }
         }
     }
 
-    fn gen_json_type(&mut self, target_type_str: &str, json_schema: &Value) -> Result<NodeRef> {
-        match target_type_str {
-            "null" => return Ok(self.builder.string("null")),
-            "boolean" => return Ok(self.lexeme(r"true|false")),
-            "integer" | "number" => {
-                let mut minimum = json_schema.opt_f64("minimum")?;
-                let mut maximum = json_schema.opt_f64("maximum")?;
-                let exclusive_minimum: bool = match json_schema.get("exclusiveMinimum") {
-                    // Draft4-style boolean
-                    Some(Value::Bool(b)) => *b,
-                    // Draft 2020-12 style number
-                    Some(Value::Number(n)) => {
-                        let n = n.as_f64()
-                            .ok_or_else(|| expected_err("exclusiveMinimum", &Value::Number(n.clone()), "unsigned integer"))?;
-                        match minimum {
-                            Some(min) if n < min => {
-                                false
-                            },
-                            _ => {
-                                minimum = Some(n);
-                                true
-                            }
-                        }
-                    },
-                    _ => false,
-                };
-                let exclusive_maximum: bool = match json_schema.get("exclusiveMaximum") {
-                    // Draft4-style boolean
-                    Some(Value::Bool(b)) => *b,
-                    // Draft 2020-12 style number
-                    Some(Value::Number(n)) => {
-                        let n = n.as_f64()
-                            .ok_or_else(|| expected_err("exclusiveMaximum", &Value::Number(n.clone()), "unsigned integer"))?;
-                        match maximum {
-                            Some(max) if n > max => {
-                                false
-                            },
-                            _ => {
-                                maximum = Some(n);
-                                true
-                            }
-                        }
-                    },
-                    _ => false,
-                };
-                if target_type_str == "integer" {
-                    return self.json_int(minimum, maximum, exclusive_minimum, exclusive_maximum);
-                } else {
-                    return self.json_number(minimum, maximum, exclusive_minimum, exclusive_maximum);
-                }
-            },
-            "string" => {
-                let min_length = json_schema.opt_u64("minLength")?.unwrap_or(0);
-                let max_length = json_schema.opt_u64("maxLength")?;
-                let pattern = json_schema.opt_str("pattern")?;
-                let format = json_schema.opt_str("format")?;
-                return self.gen_json_string(min_length, max_length, pattern, format);
-            }
-            "array" => {
-                let empty = vec![];
-                let prefix_items = json_schema.opt_array("prefixItems")?.unwrap_or(&empty);
-                let item_schema = json_schema.get("items").unwrap_or(&Value::Bool(true));
-                let min_items = json_schema.opt_u64("minItems")?.unwrap_or(0);
-                let max_items = json_schema.opt_u64("maxItems")?;
-                return self.gen_json_array(prefix_items, item_schema, min_items, max_items);
-            }
-            "object" => {
-                let empty = serde_json::Map::default();
-                let properties = json_schema.opt_object("properties")?.unwrap_or(&empty);
-                let additional_properties = json_schema
-                    .get("additionalProperties")
-                    .unwrap_or(&Value::Bool(true));
-                let required: Vec<String> = json_schema.opt_array("required")?
-                    .map(|values| {
-                        values.iter()
-                            .map(|v| {
-                                v.as_str()
-                                    .map(|s| s.to_string()) // Convert &str to String
-                                    .ok_or_else(|| expected_err("required", v, "array of string"))
-                            })
-                            .collect::<Result<Vec<_>, _>>()
-                    })
-                    .unwrap_or_else(|| Ok(vec![]))?;
-                return self.gen_json_object(properties, additional_properties, required);
-            }
-            _ => bail!("Unsupported type in schema: {}", target_type_str),
-        }
-    }
-
     fn lexeme(&mut self, rx: &str) -> NodeRef {
         if self.lexeme_cache.contains_key(rx) {
             return self.lexeme_cache[rx];
@@ -592,27 +382,19 @@ impl Compiler {
         Ok(r)
     }
 
-    fn gen_json_any(&mut self) -> NodeRef {
+    fn gen_json_any(&mut self) -> Result<NodeRef> {
         cache!(self.any_cache, {
             let json_any = self.builder.placeholder();
             self.any_cache = Some(json_any); // avoid infinite recursion
-            let all_jsons = json!([
-                {"type": "null"},
-                {"type": "boolean"},
-                {"type": "integer"},
-                {"type": "number"},
-                {"type": "string"},
-                {"type": "array", "items": true},
-                {"type": "object", "additionalProperties": true},
-            ]);
-            let ch = all_jsons
-                .as_array()
-                .unwrap()
-                .iter()
-                .map(|json_schema| self.gen_json(json_schema))
-                .collect::<Result<Vec<_>>>()
-                .unwrap();
-            let inner = self.builder.select(&ch);
+            let options = vec![
+                self.builder.string("null"),
+                self.builder.lexeme(mk_regex(r"true|false"), false),
+                self.json_number(None, None, false, false)?,
+                self.json_simple_string(),
+                self.gen_json_array(&[], &Schema::Any, 0, None)?,
+                self.gen_json_object(&IndexMap::new(), &Schema::Any, vec![])?,
+            ];
+            let inner = self.builder.select(&options);
             self.builder.set_placeholder(json_any, inner);
             json_any
         });
@@ -620,8 +402,8 @@ impl Compiler {
 
     fn gen_json_object(
         &mut self,
-        properties: &serde_json::Map<String, Value>,
-        additional_properties: &Value,
+        properties: &IndexMap<String, Schema>,
+        additional_properties: &Schema,
         required: Vec<String>,
     ) -> Result<NodeRef> {
         let mut taken_names: Vec<String> = vec![];
@@ -799,8 +581,8 @@ impl Compiler {
 
     fn gen_json_array(
         &mut self,
-        prefix_items: &[Value],
-        item_schema: &Value,
+        prefix_items: &[Schema],
+        item_schema: &Schema,
         min_items: u64,
         max_items: Option<u64>,
     ) -> Result<NodeRef> {
@@ -840,51 +622,45 @@ impl Compiler {
             max as usize
         });
 
-        if let Some(item_arr) = item_schema.as_array() {
-            for item in item_arr {
-                required_items.push(self.gen_json(item)?);
-            }
-        } else {
-            for i in 0..n_to_add {
-                let item = if i < prefix_items.len() {
-                    match self.gen_json(&prefix_items[i]) {
-                        Ok(node) => node,
-                        Err(e) => match e.downcast_ref::<UnsatisfiableSchemaError>() {
-                            // If it's not an UnsatisfiableSchemaError, just propagate it normally
-                            None => { return Err(e) },
-                            // Item is optional; don't raise UnsatisfiableSchemaError.
-                            // Set max_items to the current index, as we can't satisfy any more items.
-                            Some(_) if i >= min_items as usize => {
-                                max_items = Some(i as u64);
-                                break;
-                            },
-                            // Item is required; add context and propagate UnsatisfiableSchemaError
-                            Some(_) => {
-                                return Err(e.context(UnsatisfiableSchemaError {
-                                    message: format!("prefixItems[{}] is unsatisfiable but minItems is {}", i, min_items),
-                                }));
-                            }
+        for i in 0..n_to_add {
+            let item = if i < prefix_items.len() {
+                match self.gen_json(&prefix_items[i]) {
+                    Ok(node) => node,
+                    Err(e) => match e.downcast_ref::<UnsatisfiableSchemaError>() {
+                        // If it's not an UnsatisfiableSchemaError, just propagate it normally
+                        None => { return Err(e) },
+                        // Item is optional; don't raise UnsatisfiableSchemaError.
+                        // Set max_items to the current index, as we can't satisfy any more items.
+                        Some(_) if i >= min_items as usize => {
+                            max_items = Some(i as u64);
+                            break;
+                        },
+                        // Item is required; add context and propagate UnsatisfiableSchemaError
+                        Some(_) => {
+                            return Err(e.context(UnsatisfiableSchemaError {
+                                message: format!("prefixItems[{}] is unsatisfiable but minItems is {}", i, min_items),
+                            }));
                         }
                     }
-                } else if let Some(compiled) = &additional_item_grm {
-                    compiled.clone()
-                } else {
-                    break;
-                };
-
-                if i < min_items as usize {
-                    required_items.push(item);
-                } else {
-                    optional_items.push(item);
                 }
-            }
+            } else if let Some(compiled) = &additional_item_grm {
+                compiled.clone()
+            } else {
+                break;
+            };
 
-            if max_items.is_none() && !additional_item_grm.is_none() {
-                // Add an infinite tail of items
-                optional_items.push(self.sequence(additional_item_grm.unwrap()));
+            if i < min_items as usize {
+                required_items.push(item);
+            } else {
+                optional_items.push(item);
             }
         }
 
+        if max_items.is_none() && !additional_item_grm.is_none() {
+            // Add an infinite tail of items
+            optional_items.push(self.sequence(additional_item_grm.unwrap()));
+        }
+
         let mut grammars: Vec<NodeRef> = vec![self.builder.string("[")];
         let comma = self.builder.string(&self.options.item_separator);
 

From fb8744eb0a1ec296533fc7358a3e5e0cd9c17070 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 15 Nov 2024 16:50:36 -0800
Subject: [PATCH 52/73] cargo fmt

---
 parser/build.rs             |   5 +-
 parser/src/json/compiler.rs | 181 +++++++++++++++++++++++++-----------
 parser/src/json/formats.rs  |  47 ++++++++--
 parser/src/json/mod.rs      |   2 +-
 parser/src/json/numeric.rs  |  24 +++--
 parser/src/json/schema.rs   |  11 ++-
 parser/src/tokenparser.rs   |   2 +-
 7 files changed, 189 insertions(+), 83 deletions(-)

diff --git a/parser/build.rs b/parser/build.rs
index 8b804f0..69e1b6a 100644
--- a/parser/build.rs
+++ b/parser/build.rs
@@ -25,7 +25,10 @@ fn main() {
             },
             |bindings| {
                 bindings.write_to_file("llguidance.h");
-                bindings.write_to_file(format!("{}/../../../llguidance.h", env::var("OUT_DIR").unwrap()));
+                bindings.write_to_file(format!(
+                    "{}/../../../llguidance.h",
+                    env::var("OUT_DIR").unwrap()
+                ));
             },
         );
 }
diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index 3f866a0..02e48d4 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -3,9 +3,9 @@ use indexmap::IndexMap;
 use serde_json::{json, Value};
 use std::{collections::HashMap, vec};
 
-use super::schema::{build_schema, Schema};
 use super::formats::lookup_format;
 use super::numeric::{rx_float_range, rx_int_range};
+use super::schema::{build_schema, Schema};
 use crate::{
     api::{GrammarWithLexer, RegexSpec, TopLevelGrammar},
     GrammarBuilder, NodeRef,
@@ -37,7 +37,9 @@ impl std::fmt::Display for UnsatisfiableSchemaError {
     }
 }
 
-const TYPES: [&str; 7] = ["null", "boolean", "integer", "number", "string", "array", "object"];
+const TYPES: [&str; 7] = [
+    "null", "boolean", "integer", "number", "string", "array", "object",
+];
 const KEYWORDS: [&str; 10] = [
     "anyOf",
     "oneOf",
@@ -89,7 +91,14 @@ fn validate_json_node_keys(node: &Value) -> Result<()> {
 
     for key in node.keys() {
         let key = &key.as_str();
-        if KEYWORDS.contains(key) || IGNORED_KEYS.contains(key) || DEFS_KEYS.contains(key) || ARRAY_KEYS.contains(key) || OBJECT_KEYS.contains(key) || STRING_KEYS.contains(key) || NUMERIC_KEYS.contains(key) {
+        if KEYWORDS.contains(key)
+            || IGNORED_KEYS.contains(key)
+            || DEFS_KEYS.contains(key)
+            || ARRAY_KEYS.contains(key)
+            || OBJECT_KEYS.contains(key)
+            || STRING_KEYS.contains(key)
+            || NUMERIC_KEYS.contains(key)
+        {
             continue;
         }
         if key.starts_with("x-") || key.starts_with("$xsd-") {
@@ -101,7 +110,12 @@ fn validate_json_node_keys(node: &Value) -> Result<()> {
     Ok(())
 }
 
-fn check_number_bounds(minimum: Option<f64>, maximum: Option<f64>, exclusive_minimum: bool, exclusive_maximum: bool) -> Result<()> {
+fn check_number_bounds(
+    minimum: Option<f64>,
+    maximum: Option<f64>,
+    exclusive_minimum: bool,
+    exclusive_maximum: bool,
+) -> Result<()> {
     if let (Some(min), Some(max)) = (minimum, maximum) {
         if min > max {
             return Err(anyhow!(UnsatisfiableSchemaError {
@@ -109,10 +123,21 @@ fn check_number_bounds(minimum: Option<f64>, maximum: Option<f64>, exclusive_min
             }));
         }
         if min == max && (exclusive_minimum || exclusive_maximum) {
-            let minimum_repr = if exclusive_minimum { "exclusiveMinimum" } else { "minimum" };
-            let maximum_repr = if exclusive_maximum { "exclusiveMaximum" } else { "maximum" };
+            let minimum_repr = if exclusive_minimum {
+                "exclusiveMinimum"
+            } else {
+                "minimum"
+            };
+            let maximum_repr = if exclusive_maximum {
+                "exclusiveMaximum"
+            } else {
+                "maximum"
+            };
             return Err(anyhow!(UnsatisfiableSchemaError {
-                message: format!("{} ({}) is equal to {} ({})", minimum_repr, min, maximum_repr, max),
+                message: format!(
+                    "{} ({}) is equal to {} ({})",
+                    minimum_repr, min, maximum_repr, max
+                ),
             }));
         }
     }
@@ -230,8 +255,8 @@ impl Compiler {
                 minimum,
                 maximum,
                 exclusive_minimum,
-                exclusive_maximum ,
-                integer
+                exclusive_maximum,
+                integer,
             } => {
                 let (minimum, exclusive_minimum) = match (minimum, exclusive_minimum) {
                     (Some(min), Some(xmin)) => {
@@ -262,24 +287,38 @@ impl Compiler {
                 } else {
                     self.json_number(minimum, maximum, exclusive_minimum, exclusive_maximum)
                 }
-            },
+            }
             Schema::String {
                 min_length,
                 max_length,
                 pattern,
                 format,
-            } => self.gen_json_string(*min_length, *max_length, pattern.as_deref(), format.as_deref()),
+            } => self.gen_json_string(
+                *min_length,
+                *max_length,
+                pattern.as_deref(),
+                format.as_deref(),
+            ),
             Schema::Array {
                 min_items,
                 max_items,
                 prefix_items,
                 items,
-            } => self.gen_json_array(prefix_items, items.as_deref().unwrap_or(&true.into()), *min_items, *max_items),
+            } => self.gen_json_array(
+                prefix_items,
+                items.as_deref().unwrap_or(&true.into()),
+                *min_items,
+                *max_items,
+            ),
             Schema::Object {
                 properties,
                 additional_properties,
                 required,
-            } => self.gen_json_object(properties, additional_properties.as_deref().unwrap_or(&true.into()), required.iter().cloned().collect()),
+            } => self.gen_json_object(
+                properties,
+                additional_properties.as_deref().unwrap_or(&true.into()),
+                required.iter().cloned().collect(),
+            ),
             Schema::Const { value } => self.gen_json_const(value.clone()),
             Schema::Enum { options } => self.gen_json_enum(options.clone()),
             Schema::AnyOf { options } => self.process_any_of(options.clone()),
@@ -289,14 +328,16 @@ impl Compiler {
     }
 
     fn process_any_of(&mut self, options: Vec<Schema>) -> Result<NodeRef> {
-        let options = options.iter()
+        let options = options
+            .iter()
             .map(|v| self.gen_json(v))
             .collect::<Result<Vec<_>>>()?;
         Ok(self.builder.select(&options))
     }
 
     fn gen_json_enum(&mut self, options: Vec<Value>) -> Result<NodeRef> {
-        let options = options.into_iter()
+        let options = options
+            .into_iter()
             .map(|v| self.gen_json_const(v))
             .collect::<Result<Vec<_>>>()?;
         Ok(self.builder.select(&options))
@@ -306,15 +347,22 @@ impl Compiler {
         // Recursively build a grammar for a constant value (just to play nicely with separators and whitespace flexibility)
         match const_value {
             Value::Object(values) => {
-                let properties = IndexMap::from_iter(values.into_iter().map(|(k, v)| (k, Schema::Const { value: v })));
+                let properties = IndexMap::from_iter(
+                    values
+                        .into_iter()
+                        .map(|(k, v)| (k, Schema::Const { value: v })),
+                );
                 let required = properties.keys().cloned().collect();
                 self.gen_json_object(&properties, &false.into(), required)
-            },
+            }
             Value::Array(values) => {
                 let n_items = values.len() as u64;
-                let prefix_items = values.into_iter().map(|v| Schema::Const { value: v }).collect::<Vec<_>>();
+                let prefix_items = values
+                    .into_iter()
+                    .map(|v| Schema::Const { value: v })
+                    .collect::<Vec<_>>();
                 self.gen_json_array(&prefix_items, &false.into(), n_items, Some(n_items))
-            },
+            }
             _ => {
                 // let serde_json dump simple values
                 let const_str = json_dumps(&const_value);
@@ -332,7 +380,13 @@ impl Compiler {
         r
     }
 
-    fn json_int(&mut self, minimum: Option<f64>, maximum: Option<f64>, exclusive_minimum: bool, exclusive_maximum: bool) -> Result<NodeRef> {
+    fn json_int(
+        &mut self,
+        minimum: Option<f64>,
+        maximum: Option<f64>,
+        exclusive_minimum: bool,
+        exclusive_maximum: bool,
+    ) -> Result<NodeRef> {
         check_number_bounds(minimum, maximum, exclusive_minimum, exclusive_maximum)?;
         let minimum = match (minimum, exclusive_minimum) {
             (Some(min_val), true) => {
@@ -344,7 +398,8 @@ impl Compiler {
             }
             (Some(min_val), false) => Some(min_val.ceil()),
             _ => None,
-        }.map(|val| val as i64);
+        }
+        .map(|val| val as i64);
         let maximum = match (maximum, exclusive_maximum) {
             (Some(max_val), true) => {
                 if max_val.fract() != 0.0 {
@@ -352,16 +407,23 @@ impl Compiler {
                 } else {
                     Some(max_val - 1.0)
                 }
-            },
+            }
             (Some(max_val), false) => Some(max_val.floor()),
             _ => None,
-        }.map(|val| val as i64);
+        }
+        .map(|val| val as i64);
         // TODO: handle errors in rx_int_range; currently it just panics
         let rx = rx_int_range(minimum, maximum);
         Ok(self.lexeme(&rx))
     }
 
-    fn json_number(&mut self, minimum: Option<f64>, maximum: Option<f64>, exclusive_minimum: bool, exclusive_maximum: bool) -> Result<NodeRef> {
+    fn json_number(
+        &mut self,
+        minimum: Option<f64>,
+        maximum: Option<f64>,
+        exclusive_minimum: bool,
+        exclusive_maximum: bool,
+    ) -> Result<NodeRef> {
         check_number_bounds(minimum, maximum, exclusive_minimum, exclusive_maximum)?;
         // TODO: handle errors in rx_float_range; currently it just panics
         let rx = rx_float_range(minimum, maximum, !exclusive_minimum, !exclusive_maximum);
@@ -408,7 +470,11 @@ impl Compiler {
     ) -> Result<NodeRef> {
         let mut taken_names: Vec<String> = vec![];
         let mut items: Vec<(NodeRef, bool)> = vec![];
-        for name in properties.keys().chain(required.iter().filter(|n| !properties.contains_key(n.as_str()))) {
+        for name in properties.keys().chain(
+            required
+                .iter()
+                .filter(|n| !properties.contains_key(n.as_str())),
+        ) {
             let property_schema = properties.get(name).unwrap_or(additional_properties);
             let is_required = required.contains(name);
             // Quote (and escape) the name
@@ -417,19 +483,19 @@ impl Compiler {
                 Ok(node) => node,
                 Err(e) => match e.downcast_ref::<UnsatisfiableSchemaError>() {
                     // If it's not an UnsatisfiableSchemaError, just propagate it normally
-                    None => { return Err(e) },
+                    None => return Err(e),
                     // Property is optional; don't raise UnsatisfiableSchemaError but mark name as taken
                     Some(_) if !is_required => {
                         taken_names.push(quoted_name);
                         continue;
-                    },
+                    }
                     // Property is required; add context and propagate UnsatisfiableSchemaError
                     Some(_) => {
                         return Err(e.context(UnsatisfiableSchemaError {
                             message: format!("required property '{}' is unsatisfiable", name),
                         }));
                     }
-                }
+                },
             };
             let name = self.builder.string(&quoted_name);
             taken_names.push(quoted_name);
@@ -445,19 +511,21 @@ impl Compiler {
                     return Err(e);
                 }
                 // Ignore UnsatisfiableSchemaError for additionalProperties
-            },
+            }
             Ok(property) => {
                 let name = if taken_names.is_empty() {
                     self.json_simple_string()
                 } else {
-                    let taken_name_ids =
-                        taken_names
+                    let taken_name_ids = taken_names
                         .iter()
                         .map(|n| self.builder.regex.literal(n.to_string()))
                         .collect::<Vec<_>>();
                     let taken = self.builder.regex.select(taken_name_ids);
                     let not_taken = self.builder.regex.not(taken);
-                    let valid = self.builder.regex.regex(r#""([^"\\]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})*""#.to_string());
+                    let valid = self
+                        .builder
+                        .regex
+                        .regex(r#""([^"\\]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})*""#.to_string());
                     let valid_and_not_taken = self.builder.regex.and(vec![valid, not_taken]);
                     let rx = RegexSpec::RegexId(valid_and_not_taken);
                     self.builder.lexeme(rx, false)
@@ -474,7 +542,12 @@ impl Compiler {
         Ok(self.builder.join(&[opener, inner, closer]))
     }
 
-    fn ordered_sequence<'a>(&mut self, items: &'a[(NodeRef, bool)], prefixed: bool, cache: &mut HashMap::<(&'a[(NodeRef, bool)], bool), NodeRef>) -> NodeRef {
+    fn ordered_sequence<'a>(
+        &mut self,
+        items: &'a [(NodeRef, bool)],
+        prefixed: bool,
+        cache: &mut HashMap<(&'a [(NodeRef, bool)], bool), NodeRef>,
+    ) -> NodeRef {
         // Cache to reduce number of nodes from O(n^2) to O(n)
         if let Some(node) = cache.get(&(items, prefixed)) {
             return node.clone();
@@ -491,7 +564,7 @@ impl Compiler {
                 // If we know we have preceeding elements, we can safely just add a (',' + e)
                 let rest_seq = self.ordered_sequence(rest, true, cache);
                 self.builder.join(&[comma, item, rest_seq])
-            },
+            }
             (true, false) => {
                 // If we know we have preceeding elements, we can safely just add an optional(',' + e)
                 // TODO optimization: if the rest is all optional, we can nest the rest in the optional
@@ -499,24 +572,21 @@ impl Compiler {
                 let optional_comma_item = self.builder.optional(comma_item);
                 let rest_seq = self.ordered_sequence(rest, true, cache);
                 self.builder.join(&[optional_comma_item, rest_seq])
-            },
+            }
             (false, true) => {
                 // No preceding elements, so we just add the element (no comma)
                 let rest_seq = self.ordered_sequence(rest, true, cache);
                 self.builder.join(&[item, rest_seq])
-            },
+            }
             (false, false) => {
                 // No preceding elements, but our element is optional. If we add the element, the remaining
                 // will be prefixed, else they are not.
                 // TODO: same nested optimization as above
                 let prefixed_rest = self.ordered_sequence(rest, true, cache);
                 let unprefixed_rest = self.ordered_sequence(rest, false, cache);
-                let opts = [
-                    self.builder.join(&[item, prefixed_rest]),
-                    unprefixed_rest,
-                ];
+                let opts = [self.builder.join(&[item, prefixed_rest]), unprefixed_rest];
                 self.builder.select(&opts)
-            },
+            }
         };
         cache.insert((items, prefixed), node.clone());
         node
@@ -539,7 +609,10 @@ impl Compiler {
         if let Some(max_length) = max_length {
             if min_length > max_length {
                 return Err(anyhow!(UnsatisfiableSchemaError {
-                    message: format!("minLength ({}) is greater than maxLength ({})", min_length, max_length),
+                    message: format!(
+                        "minLength ({}) is greater than maxLength ({})",
+                        min_length, max_length
+                    ),
                 }));
             }
         }
@@ -591,7 +664,10 @@ impl Compiler {
         if let Some(max_items) = max_items {
             if min_items > max_items {
                 return Err(anyhow!(UnsatisfiableSchemaError {
-                    message: format!("minItems ({}) is greater than maxItems ({})", min_items, max_items),
+                    message: format!(
+                        "minItems ({}) is greater than maxItems ({})",
+                        min_items, max_items
+                    ),
                 }));
             }
         }
@@ -600,18 +676,16 @@ impl Compiler {
             Ok(node) => Some(node),
             Err(e) => match e.downcast_ref::<UnsatisfiableSchemaError>() {
                 // If it's not an UnsatisfiableSchemaError, just propagate it normally
-                None => { return Err(e) },
+                None => return Err(e),
                 // Item is optional; don't raise UnsatisfiableSchemaError
-                Some(_) if prefix_items.len() >= min_items as usize => {
-                    None
-                },
+                Some(_) if prefix_items.len() >= min_items as usize => None,
                 // Item is required; add context and propagate UnsatisfiableSchemaError
                 Some(_) => {
                     return Err(e.context(UnsatisfiableSchemaError {
                         message: format!("required item is unsatisfiable"),
                     }));
                 }
-            }
+            },
         };
 
         let mut required_items = vec![];
@@ -628,20 +702,23 @@ impl Compiler {
                     Ok(node) => node,
                     Err(e) => match e.downcast_ref::<UnsatisfiableSchemaError>() {
                         // If it's not an UnsatisfiableSchemaError, just propagate it normally
-                        None => { return Err(e) },
+                        None => return Err(e),
                         // Item is optional; don't raise UnsatisfiableSchemaError.
                         // Set max_items to the current index, as we can't satisfy any more items.
                         Some(_) if i >= min_items as usize => {
                             max_items = Some(i as u64);
                             break;
-                        },
+                        }
                         // Item is required; add context and propagate UnsatisfiableSchemaError
                         Some(_) => {
                             return Err(e.context(UnsatisfiableSchemaError {
-                                message: format!("prefixItems[{}] is unsatisfiable but minItems is {}", i, min_items),
+                                message: format!(
+                                    "prefixItems[{}] is unsatisfiable but minItems is {}",
+                                    i, min_items
+                                ),
                             }));
                         }
-                    }
+                    },
                 }
             } else if let Some(compiled) = &additional_item_grm {
                 compiled.clone()
diff --git a/parser/src/json/formats.rs b/parser/src/json/formats.rs
index e8924d3..3ec381c 100644
--- a/parser/src/json/formats.rs
+++ b/parser/src/json/formats.rs
@@ -4,16 +4,43 @@ use std::collections::HashMap;
 lazy_static! {
     static ref FORMAT_PATTERNS: HashMap<&'static str, &'static str> = {
         HashMap::from([
-            ("date-time", r"(?P<date>[0-9]{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01]))[tT](?P<time>(?:[01][0-9]|2[0-3]):[0-5][0-9]:(?:[0-5][0-9]|60)(?P<time_fraction>\.[0-9]+)?(?P<time_zone>[zZ]|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9]))"),
-            ("time", r"(?:[01][0-9]|2[0-3]):[0-5][0-9]:(?:[0-5][0-9]|60)(?P<time_fraction>\.[0-9]+)?(?P<time_zone>[zZ]|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9])"),
-            ("date", r"[0-9]{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])"),
-            ("duration", r"P(?:(?P<dur_date>(?:(?P<dur_year>[0-9]+Y(?:[0-9]+M(?:[0-9]+D)?)?)|(?P<dur_month>[0-9]+M(?:[0-9]+D)?)|(?P<dur_day>[0-9]+D))(?:T(?:(?P<dur_hour>[0-9]+H(?:[0-9]+M(?:[0-9]+S)?)?)|(?P<dur_minute>[0-9]+M(?:[0-9]+S)?)|(?P<dur_second>[0-9]+S)))?)|(?P<dur_time>T(?:(?P<dur_hour2>[0-9]+H(?:[0-9]+M(?:[0-9]+S)?)?)|(?P<dur_minute2>[0-9]+M(?:[0-9]+S)?)|(?P<dur_second2>[0-9]+S)))|(?P<dur_week>[0-9]+W))"),
-            ("email", r"(?P<local_part>(?P<dot_string>[^\s@\.]+(\.[^\s@\.]+)*))@((?P<domain>(?P<sub_domain>[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?)(\.(?P<sub_domain2>[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?))*)|\[(?P<ipv4>((([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9])))\.){3}(([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9]))))\])"),
-            ("hostname", r"[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*"),
-            ("ipv4", r"((([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9])))\.){3}(([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9])))"),
-            ("ipv6", r"(?:(?P<full>(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}))|(?:::(?:[0-9a-fA-F]{1,4}:){0,5}(?P<ls32>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:(?P<h16_1>[0-9a-fA-F]{1,4})?::(?:[0-9a-fA-F]{1,4}:){0,4}(?P<ls32_1>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,1}[0-9a-fA-F]{1,4})?::(?:[0-9a-fA-F]{1,4}:){0,3}(?P<ls32_2>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,2}[0-9a-fA-F]{1,4})?::(?:[0-9a-fA-F]{1,4}:){0,2}(?P<ls32_3>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,3}[0-9a-fA-F]{1,4})?::[0-9a-fA-F]{1,4}:(?P<ls32_4>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,4}[0-9a-fA-F]{1,4})?::(?P<ls32_5>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,5}[0-9a-fA-F]{1,4})?::(?P<h16_2>[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})?::)"),
-            ("uuid", r"(?P<time_low>[0-9a-fA-F]{8})-(?P<time_mid>[0-9a-fA-F]{4})-(?P<time_high_and_version>[0-9a-fA-F]{4})-(?P<clock_seq_and_reserved>[0-9a-fA-F]{2})(?P<clock_seq_low>[0-9a-fA-F]{2})-(?P<node>[0-9a-fA-F]{12})"),
-            ("unknown", r"(?s:.*)")
+            (
+                "date-time",
+                r"(?P<date>[0-9]{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01]))[tT](?P<time>(?:[01][0-9]|2[0-3]):[0-5][0-9]:(?:[0-5][0-9]|60)(?P<time_fraction>\.[0-9]+)?(?P<time_zone>[zZ]|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9]))",
+            ),
+            (
+                "time",
+                r"(?:[01][0-9]|2[0-3]):[0-5][0-9]:(?:[0-5][0-9]|60)(?P<time_fraction>\.[0-9]+)?(?P<time_zone>[zZ]|[+-](?:[01][0-9]|2[0-3]):[0-5][0-9])",
+            ),
+            (
+                "date",
+                r"[0-9]{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])",
+            ),
+            (
+                "duration",
+                r"P(?:(?P<dur_date>(?:(?P<dur_year>[0-9]+Y(?:[0-9]+M(?:[0-9]+D)?)?)|(?P<dur_month>[0-9]+M(?:[0-9]+D)?)|(?P<dur_day>[0-9]+D))(?:T(?:(?P<dur_hour>[0-9]+H(?:[0-9]+M(?:[0-9]+S)?)?)|(?P<dur_minute>[0-9]+M(?:[0-9]+S)?)|(?P<dur_second>[0-9]+S)))?)|(?P<dur_time>T(?:(?P<dur_hour2>[0-9]+H(?:[0-9]+M(?:[0-9]+S)?)?)|(?P<dur_minute2>[0-9]+M(?:[0-9]+S)?)|(?P<dur_second2>[0-9]+S)))|(?P<dur_week>[0-9]+W))",
+            ),
+            (
+                "email",
+                r"(?P<local_part>(?P<dot_string>[^\s@\.]+(\.[^\s@\.]+)*))@((?P<domain>(?P<sub_domain>[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?)(\.(?P<sub_domain2>[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?))*)|\[(?P<ipv4>((([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9])))\.){3}(([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9]))))\])",
+            ),
+            (
+                "hostname",
+                r"[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*",
+            ),
+            (
+                "ipv4",
+                r"((([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9])))\.){3}(([0-9])|(([1-9])[0-9]|(25[0-5]|(2[0-4]|(1)[0-9])[0-9])))",
+            ),
+            (
+                "ipv6",
+                r"(?:(?P<full>(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}))|(?:::(?:[0-9a-fA-F]{1,4}:){0,5}(?P<ls32>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:(?P<h16_1>[0-9a-fA-F]{1,4})?::(?:[0-9a-fA-F]{1,4}:){0,4}(?P<ls32_1>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,1}[0-9a-fA-F]{1,4})?::(?:[0-9a-fA-F]{1,4}:){0,3}(?P<ls32_2>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,2}[0-9a-fA-F]{1,4})?::(?:[0-9a-fA-F]{1,4}:){0,2}(?P<ls32_3>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,3}[0-9a-fA-F]{1,4})?::[0-9a-fA-F]{1,4}:(?P<ls32_4>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,4}[0-9a-fA-F]{1,4})?::(?P<ls32_5>[0-9a-fA-F]{1,4}:[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,5}[0-9a-fA-F]{1,4})?::(?P<h16_2>[0-9a-fA-F]{1,4}))|(?:((?:[0-9a-fA-F]{1,4}:){0,6}[0-9a-fA-F]{1,4})?::)",
+            ),
+            (
+                "uuid",
+                r"(?P<time_low>[0-9a-fA-F]{8})-(?P<time_mid>[0-9a-fA-F]{4})-(?P<time_high_and_version>[0-9a-fA-F]{4})-(?P<clock_seq_and_reserved>[0-9a-fA-F]{2})(?P<clock_seq_low>[0-9a-fA-F]{2})-(?P<node>[0-9a-fA-F]{12})",
+            ),
+            ("unknown", r"(?s:.*)"),
         ])
     };
 }
diff --git a/parser/src/json/mod.rs b/parser/src/json/mod.rs
index 49e7237..e08e06f 100644
--- a/parser/src/json/mod.rs
+++ b/parser/src/json/mod.rs
@@ -1,4 +1,4 @@
 pub mod compiler;
 mod formats;
-mod schema;
 mod numeric;
+mod schema;
diff --git a/parser/src/json/numeric.rs b/parser/src/json/numeric.rs
index 75c14ef..79e8d8a 100644
--- a/parser/src/json/numeric.rs
+++ b/parser/src/json/numeric.rs
@@ -237,10 +237,7 @@ pub fn rx_float_range(
                         left_inclusive,
                         false,
                     ),
-                    format!(
-                        "[1-9][0-9]{{{},}}(\\.[0-9]+)?",
-                        num_digits(left_int_part)
-                    ),
+                    format!("[1-9][0-9]{{{},}}(\\.[0-9]+)?", num_digits(left_int_part)),
                 ])
             }
         }
@@ -258,7 +255,10 @@ pub fn rx_float_range(
                     rx_float_range(Some(0.0), Some(right), true, right_inclusive),
                 ])
             } else {
-                format!("-{}", rx_float_range(Some(-right), None, right_inclusive, false))
+                format!(
+                    "-{}",
+                    rx_float_range(Some(-right), None, right_inclusive, false)
+                )
             }
         }
         (Some(left), Some(right)) => {
@@ -273,12 +273,7 @@ pub fn rx_float_range(
                 if right < 0.0 {
                     format!(
                         "(-{})",
-                        rx_float_range(
-                            Some(-right),
-                            Some(-left),
-                            right_inclusive,
-                            left_inclusive
-                        )
+                        rx_float_range(Some(-right), Some(-left), right_inclusive, left_inclusive)
                     )
                 } else {
                     let mut parts = vec![];
@@ -294,7 +289,7 @@ pub fn rx_float_range(
                         let pos_part = rx_float_range(
                             Some(0.0),
                             Some(right),
-                            true,  // always include 0
+                            true, // always include 0
                             right_inclusive,
                         );
                         parts.push(pos_part);
@@ -324,7 +319,10 @@ pub fn rx_float_range(
                     while rd.len() < ld.len() {
                         rd.push('0');
                     }
-                    let suff = format!("\\.{}", lexi_range(&ld, &rd, left_inclusive, right_inclusive));
+                    let suff = format!(
+                        "\\.{}",
+                        lexi_range(&ld, &rd, left_inclusive, right_inclusive)
+                    );
                     if ld.parse::<i64>().unwrap() == 0 {
                         format!("({}({})?)", left_rec, suff)
                     } else {
diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index 89a3eab..12b960e 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -191,7 +191,7 @@ impl Schema {
                 Ok(Schema::OneOf { options: valid })
             }
             // TODO: ?
-            Schema::Ref{ .. } => Ok(self),
+            Schema::Ref { .. } => Ok(self),
         }
     }
 }
@@ -221,7 +221,10 @@ impl<'a> Context<'a> {
     }
 
     fn normalize_ref(&self, reference: &str) -> Result<Uri<String>> {
-        Ok(self.resolver.resolve_against(&self.resolver.base_uri().borrow(), reference)?.normalize())
+        Ok(self
+            .resolver
+            .resolve_against(&self.resolver.base_uri().borrow(), reference)?
+            .normalize())
     }
 
     fn lookup_resource(&'a self, reference: &str) -> Result<ResourceRef> {
@@ -370,9 +373,7 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
         let uri: String = ctx.normalize_ref(&reference)?.into_string();
         let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
         if siblings == Schema::Any {
-            let placeholder = Schema::Ref {
-                uri: uri.clone()
-            };
+            let placeholder = Schema::Ref { uri: uri.clone() };
             if ctx.contains_ref(uri.as_str()) {
                 // We've already built this ref, so we can just return it
                 return Ok(placeholder);
diff --git a/parser/src/tokenparser.rs b/parser/src/tokenparser.rs
index 3802d57..5f2316b 100644
--- a/parser/src/tokenparser.rs
+++ b/parser/src/tokenparser.rs
@@ -258,7 +258,7 @@ impl TokenParser {
         self.error_message.clone()
     }
 
-    // advance_parser() is a top-level method in this file.  
+    // advance_parser() is a top-level method in this file.
     // This advance_parser() is called indirectly via the advance_parser() method
     // of the llguidance LLInterpreter interface,
     //

From 1c894137645214265b84b460675b3cf28c0d6b62 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 15 Nov 2024 17:02:59 -0800
Subject: [PATCH 53/73] comment out block for a minute

---
 parser/src/json/compiler.rs | 52 ++++++++++++++++++-------------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index 02e48d4..b11f2f7 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -83,32 +83,32 @@ const NUMERIC_KEYS: [&str; 4] = ["minimum", "maximum", "exclusiveMinimum", "excl
 
 const CHAR_REGEX: &str = r#"(\\([\"\\\/bfnrt]|u[a-fA-F0-9]{4})|[^\"\\\x00-\x1F\x7F])"#;
 
-fn validate_json_node_keys(node: &Value) -> Result<()> {
-    let node = node
-        .as_object()
-        .ok_or_else(|| anyhow!("Expected object as json schema, got: {}", limited_str(node)))
-        .unwrap();
-
-    for key in node.keys() {
-        let key = &key.as_str();
-        if KEYWORDS.contains(key)
-            || IGNORED_KEYS.contains(key)
-            || DEFS_KEYS.contains(key)
-            || ARRAY_KEYS.contains(key)
-            || OBJECT_KEYS.contains(key)
-            || STRING_KEYS.contains(key)
-            || NUMERIC_KEYS.contains(key)
-        {
-            continue;
-        }
-        if key.starts_with("x-") || key.starts_with("$xsd-") {
-            continue;
-        }
-        bail!("Unknown key in JSON schema: {:?}", key);
-    }
-
-    Ok(())
-}
+// fn validate_json_node_keys(node: &Value) -> Result<()> {
+//     let node = node
+//         .as_object()
+//         .ok_or_else(|| anyhow!("Expected object as json schema, got: {}", limited_str(node)))
+//         .unwrap();
+
+//     for key in node.keys() {
+//         let key = &key.as_str();
+//         if KEYWORDS.contains(key)
+//             || IGNORED_KEYS.contains(key)
+//             || DEFS_KEYS.contains(key)
+//             || ARRAY_KEYS.contains(key)
+//             || OBJECT_KEYS.contains(key)
+//             || STRING_KEYS.contains(key)
+//             || NUMERIC_KEYS.contains(key)
+//         {
+//             continue;
+//         }
+//         if key.starts_with("x-") || key.starts_with("$xsd-") {
+//             continue;
+//         }
+//         bail!("Unknown key in JSON schema: {:?}", key);
+//     }
+
+//     Ok(())
+// }
 
 fn check_number_bounds(
     minimum: Option<f64>,

From 66df30325fae94318a1c3edda20b3a326a3e3220 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 15 Nov 2024 17:03:08 -0800
Subject: [PATCH 54/73] fix defs

---
 parser/src/json/compiler.rs | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index b11f2f7..c2cef5a 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -224,18 +224,10 @@ impl Compiler {
         let root = self.gen_json(&compiled_schema)?;
         self.builder.set_start_node(root);
 
-        while let Some((path0, pl)) = self.pending_definitions.pop() {
-            // path is #/foo/bar/baz, first split into elements
-            let path = path0.trim_start_matches("#/");
-            let path = path.split('/').collect::<Vec<_>>();
-            let mut node = schema;
-            for elem in path {
-                node = &node[elem];
-            }
-            if node.is_null() {
-                bail!("Definition not found: {}", path0);
-            }
-            let schema = definitions.get(&path0).unwrap();
+        while let Some((path, pl)) = self.pending_definitions.pop() {
+            let schema = definitions
+                .get(&path)
+                .ok_or_else(|| anyhow!("Definition not found: {}", path))?;
             let compiled = self.gen_json(schema)?;
             self.builder.set_placeholder(pl, compiled);
         }

From ea3a395122b4f089aec08bb0a4593a58d6bf7f80 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Fri, 15 Nov 2024 18:26:48 -0800
Subject: [PATCH 55/73] fix optional items

---
 parser/src/json/compiler.rs | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index c2cef5a..ab3b04a 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -743,14 +743,16 @@ impl Compiler {
 
         if !optional_items.is_empty() {
             let first = optional_items[0];
-            let tail = optional_items
-                .into_iter()
-                .skip(1)
-                .rev()
-                .fold(first, |acc, item| {
-                    let j = self.builder.join(&[comma, item, acc]);
-                    self.builder.optional(j)
-                });
+            let tail =
+                optional_items
+                    .into_iter()
+                    .skip(1)
+                    .rev()
+                    .fold(self.builder.empty(), |acc, item| {
+                        let j = self.builder.join(&[comma, item, acc]);
+                        self.builder.optional(j)
+                    });
+            let tail = self.builder.join(&[first, tail]);
 
             if !required_items.is_empty() {
                 let j = self.builder.join(&[comma, tail]);

From 6e099511e1da81942811b4feb620666f42a2fdaf Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Mon, 18 Nov 2024 10:34:34 -0800
Subject: [PATCH 56/73] validate enums and consts

---
 parser/src/json/schema.rs | 37 +++++++++++++++++++++++++++++++------
 1 file changed, 31 insertions(+), 6 deletions(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index 12b960e..a8b57b2 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -2,7 +2,7 @@ use std::{cell::RefCell, collections::HashMap, rc::Rc};
 
 use anyhow::{anyhow, bail, Result};
 use indexmap::{IndexMap, IndexSet};
-use jsonschema::Uri;
+use jsonschema::{validator_for, Uri, Validator};
 use referencing::{Draft, Registry, Resolver, ResourceRef};
 use serde_json::{Map, Value};
 
@@ -245,6 +245,14 @@ fn draft_for(value: &Value) -> Draft {
     DEFAULT_DRAFT.detect(value).unwrap_or(DEFAULT_DRAFT)
 }
 
+fn instance_if_valid<'a>(instance: &'a Value, validator: &'a Validator) -> Option<&'a Value> {
+    if validator.is_valid(instance) {
+        Some(instance)
+    } else {
+        None
+    }
+}
+
 pub fn build_schema(contents: &Value) -> Result<(Schema, HashMap<String, Schema>)> {
     if let Some(b) = contents.as_bool() {
         return Ok((b.into(), HashMap::new()));
@@ -294,20 +302,37 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
         return Ok(Schema::Any);
     }
 
+    // Short-circuit for const -- don't need to compile the rest of the schema
     if let Some(instance) = schemadict.get("const") {
-        // TODO: validate the instance against the schema, maybe returning Schema::Unsatisfiable
-        return Ok(Schema::Const {
-            value: instance.clone(),
+        let validator = validator_for(contents)?;
+        if let Some(instance) = instance_if_valid(instance, &validator) {
+            return Ok(Schema::Const {
+                value: instance.clone(),
+            });
+        }
+        return Ok(Schema::Unsatisfiable {
+            reason: format!("const instance is invalid against parent schema: {:?}", instance),
         });
     }
 
+    // Short-circuit for enum -- don't need to compile the rest of the schema
     if let Some(instances) = schemadict.get("enum") {
+        let validator: Validator = validator_for(contents)?;
         let instances = instances
             .as_array()
             .ok_or_else(|| anyhow!("enum must be an array"))?;
-        // TODO: validate the instances against the schema, maybe returning Schema::Unsatisfiable
+        let valid_instances = instances
+            .iter()
+            .filter_map(|instance| instance_if_valid(instance, &validator))
+            .cloned()
+            .collect::<Vec<_>>();
+        if valid_instances.is_empty() {
+            return Ok(Schema::Unsatisfiable {
+                reason: format!("enum instances all invalid against parent schema: {:?}", instances),
+            });
+        }
         return Ok(Schema::Enum {
-            options: instances.clone(),
+            options: valid_instances,
         });
     }
 

From 64217071635bcdf7d1518e4c795276bf4bd38625 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Mon, 18 Nov 2024 12:00:52 -0800
Subject: [PATCH 57/73] check keyword validity

---
 parser/src/json/compiler.rs | 71 ---------------------------------
 parser/src/json/schema.rs   | 78 ++++++++++++++++++++++++++++++++++++-
 2 files changed, 76 insertions(+), 73 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index ab3b04a..3ba37b9 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -37,79 +37,8 @@ impl std::fmt::Display for UnsatisfiableSchemaError {
     }
 }
 
-const TYPES: [&str; 7] = [
-    "null", "boolean", "integer", "number", "string", "array", "object",
-];
-const KEYWORDS: [&str; 10] = [
-    "anyOf",
-    "oneOf",
-    "allOf",
-    "$ref",
-    "const",
-    "enum",
-    "type",
-    "pattern",
-    "minLength",
-    "maxLength",
-];
-const IGNORED_KEYS: [&str; 19] = [
-    "$schema",
-    "$id",
-    "id",
-    "$comment",
-    "title",
-    "description",
-    "default",
-    "examples",
-    "authors",
-    "deprecationMessage",
-    "enumDescriptions",
-    "example",
-    "postActions",
-    "readOnly",
-    "markdownDescription",
-    "deprecated",
-    "dependencies",
-    "discriminator", // we hope it's part of the grammar anyways
-    "required",      // TODO: implement and remove from ignored list
-];
-// these are also just ignored
-const DEFS_KEYS: [&str; 4] = ["$defs", "definitions", "defs", "refs"];
-
-const ARRAY_KEYS: [&str; 4] = ["items", "prefixItems", "minItems", "maxItems"];
-const OBJECT_KEYS: [&str; 3] = ["properties", "additionalProperties", "required"];
-const STRING_KEYS: [&str; 4] = ["minLength", "maxLength", "pattern", "format"];
-const NUMERIC_KEYS: [&str; 4] = ["minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum"];
-
 const CHAR_REGEX: &str = r#"(\\([\"\\\/bfnrt]|u[a-fA-F0-9]{4})|[^\"\\\x00-\x1F\x7F])"#;
 
-// fn validate_json_node_keys(node: &Value) -> Result<()> {
-//     let node = node
-//         .as_object()
-//         .ok_or_else(|| anyhow!("Expected object as json schema, got: {}", limited_str(node)))
-//         .unwrap();
-
-//     for key in node.keys() {
-//         let key = &key.as_str();
-//         if KEYWORDS.contains(key)
-//             || IGNORED_KEYS.contains(key)
-//             || DEFS_KEYS.contains(key)
-//             || ARRAY_KEYS.contains(key)
-//             || OBJECT_KEYS.contains(key)
-//             || STRING_KEYS.contains(key)
-//             || NUMERIC_KEYS.contains(key)
-//         {
-//             continue;
-//         }
-//         if key.starts_with("x-") || key.starts_with("$xsd-") {
-//             continue;
-//         }
-//         bail!("Unknown key in JSON schema: {:?}", key);
-//     }
-
-//     Ok(())
-// }
-
 fn check_number_bounds(
     minimum: Option<f64>,
     maximum: Option<f64>,
diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index a8b57b2..96182a2 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -10,6 +10,58 @@ const DEFAULT_ROOT_URI: &str = "json-schema:///";
 const DEFAULT_DRAFT: Draft = Draft::Draft202012;
 const TYPES: [&str; 6] = ["null", "boolean", "number", "string", "array", "object"];
 
+// Keywords that are implemented in this module
+const IMPLEMENTED: [&str; 22] = [
+    // Core
+    "anyOf",
+    "oneOf",
+    "allOf",
+    "$ref",
+    "const",
+    "enum",
+    "type",
+    // Array
+    "items",
+    "prefixItems",
+    "minItems",
+    "maxItems",
+    // Object
+    "properties",
+    "additionalProperties",
+    "required",
+    // String
+    "minLength",
+    "maxLength",
+    "pattern",
+    "format",
+    // Number
+    "minimum",
+    "maximum",
+    "exclusiveMinimum",
+    "exclusiveMaximum",
+];
+
+// Keywords that are used for metadata or annotations, not directly driving validation.
+// Note that some keywords like $id and $schema affect the behavior of other keywords, but
+// they can safely be ignored if other keywords aren't present
+const META_AND_ANNOTATIONS: [&str; 15] = [
+    "$anchor",
+    "$defs",
+    "definitions",
+    "$schema",
+    "$id",
+    "id",
+    "$comment",
+    "title",
+    "description",
+    "default",
+    "readOnly",
+    "writeOnly",
+    "examples",
+    "contentMediaType",
+    "contentEncoding",
+];
+
 fn limited_str(node: &Value) -> String {
     let s = node.to_string();
     if s.len() > 100 {
@@ -239,6 +291,16 @@ impl<'a> Context<'a> {
     fn contains_ref(&self, uri: &str) -> bool {
         self.defs.borrow().contains_key(uri)
     }
+
+    fn is_valid_keyword(&self, keyword: &str) -> bool {
+        if !self.draft.is_known_keyword(keyword)
+            || IMPLEMENTED.contains(&keyword)
+            || META_AND_ANNOTATIONS.contains(&keyword)
+        {
+            return true;
+        }
+        return false;
+    }
 }
 
 fn draft_for(value: &Value) -> Draft {
@@ -297,11 +359,23 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
         .as_object()
         .ok_or_else(|| anyhow!("schema must be an object or boolean"))?;
 
-    if schemadict.is_empty() {
-        // TODO: should be ok to have ignored keys here
+    // We don't need to compile the schema if it's just meta and annotations
+    if schemadict
+        .keys()
+        .all(|k| META_AND_ANNOTATIONS.contains(&k.as_str()))
+    {
         return Ok(Schema::Any);
     }
 
+    // Check for unimplemented keys and bail if any are found
+    let unimplemented_keys = schemadict
+        .keys()
+        .filter(|k| !ctx.is_valid_keyword(k))
+        .collect::<Vec<_>>();
+    if unimplemented_keys.len() > 0 {
+        bail!("Unimplemented keys: {:?}", unimplemented_keys);
+    }
+
     // Short-circuit for const -- don't need to compile the rest of the schema
     if let Some(instance) = schemadict.get("const") {
         let validator = validator_for(contents)?;

From f050eba8a8484108b07fdfea67b57eaa5559f466 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Mon, 18 Nov 2024 12:01:49 -0800
Subject: [PATCH 58/73] cargo fmt

---
 parser/src/json/schema.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index 96182a2..033dca7 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -385,7 +385,10 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
             });
         }
         return Ok(Schema::Unsatisfiable {
-            reason: format!("const instance is invalid against parent schema: {:?}", instance),
+            reason: format!(
+                "const instance is invalid against parent schema: {:?}",
+                instance
+            ),
         });
     }
 
@@ -402,7 +405,10 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
             .collect::<Vec<_>>();
         if valid_instances.is_empty() {
             return Ok(Schema::Unsatisfiable {
-                reason: format!("enum instances all invalid against parent schema: {:?}", instances),
+                reason: format!(
+                    "enum instances all invalid against parent schema: {:?}",
+                    instances
+                ),
             });
         }
         return Ok(Schema::Enum {

From 0e971ccbec511d3650ed79d8be38e958a0a31724 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Mon, 18 Nov 2024 16:18:18 -0800
Subject: [PATCH 59/73] make @mmoskal less sad

---
 parser/src/json/compiler.rs |  8 ++++----
 parser/src/json/schema.rs   | 26 ++++++++++++++------------
 2 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index 3ba37b9..e30a653 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -227,7 +227,7 @@ impl Compiler {
                 items,
             } => self.gen_json_array(
                 prefix_items,
-                items.as_deref().unwrap_or(&true.into()),
+                items.as_deref().unwrap_or(&Schema::Any),
                 *min_items,
                 *max_items,
             ),
@@ -237,7 +237,7 @@ impl Compiler {
                 required,
             } => self.gen_json_object(
                 properties,
-                additional_properties.as_deref().unwrap_or(&true.into()),
+                additional_properties.as_deref().unwrap_or(&Schema::Any),
                 required.iter().cloned().collect(),
             ),
             Schema::Const { value } => self.gen_json_const(value.clone()),
@@ -274,7 +274,7 @@ impl Compiler {
                         .map(|(k, v)| (k, Schema::Const { value: v })),
                 );
                 let required = properties.keys().cloned().collect();
-                self.gen_json_object(&properties, &false.into(), required)
+                self.gen_json_object(&properties, &Schema::false_schema(), required)
             }
             Value::Array(values) => {
                 let n_items = values.len() as u64;
@@ -282,7 +282,7 @@ impl Compiler {
                     .into_iter()
                     .map(|v| Schema::Const { value: v })
                     .collect::<Vec<_>>();
-                self.gen_json_array(&prefix_items, &false.into(), n_items, Some(n_items))
+                self.gen_json_array(&prefix_items, &Schema::false_schema(), n_items, Some(n_items))
             }
             _ => {
                 // let serde_json dump simple values
diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index 033dca7..199d5c5 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -120,19 +120,13 @@ pub enum Schema {
     },
 }
 
-impl From<bool> for Schema {
-    fn from(b: bool) -> Schema {
-        if b {
-            Schema::Any
-        } else {
-            Schema::Unsatisfiable {
-                reason: "Schema is false".to_string(),
-            }
+impl Schema {
+    pub fn false_schema() -> Schema {
+        Schema::Unsatisfiable {
+            reason: "Schema is false".to_string(),
         }
     }
-}
 
-impl Schema {
     fn normalize(self) -> Result<Schema> {
         match self {
             Schema::Any => Ok(self),
@@ -317,7 +311,11 @@ fn instance_if_valid<'a>(instance: &'a Value, validator: &'a Validator) -> Optio
 
 pub fn build_schema(contents: &Value) -> Result<(Schema, HashMap<String, Schema>)> {
     if let Some(b) = contents.as_bool() {
-        return Ok((b.into(), HashMap::new()));
+        if b {
+            return Ok((Schema::Any, HashMap::new()));
+        } else {
+            return Ok((Schema::false_schema(), HashMap::new()));
+        }
     }
 
     let draft = draft_for(contents);
@@ -350,7 +348,11 @@ fn compile_contents(ctx: &Context, contents: &Value) -> Result<Schema> {
 
 fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
     if let Some(b) = contents.as_bool() {
-        return Ok(b.into());
+        if b {
+            return Ok(Schema::Any);
+        } else {
+            return Ok(Schema::false_schema());
+        }
     }
 
     // Get the schema as an object

From be74d8986091e60b7da30fb12fecbe6ecb3266e9 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Mon, 18 Nov 2024 18:29:44 -0800
Subject: [PATCH 60/73] rename merge -> intersect and have it take ownership

---
 parser/src/json/schema.rs | 177 ++++++++++++++++++--------------------
 1 file changed, 84 insertions(+), 93 deletions(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index 199d5c5..257ad90 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -434,7 +434,7 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
             .iter()
             .map(|value| compile_resource(&ctx, ctx.as_resource_ref(value)))
             .collect::<Result<Vec<_>>>()?;
-        let merged = merge(options.iter().chain(vec![&siblings]).collect())?;
+        let merged = intersect(options.into_iter().chain(vec![siblings]).collect())?;
         return Ok(merged);
     }
 
@@ -448,8 +448,9 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
             return Ok(siblings);
         }
         let options = any_of
-            .iter()
+            .into_iter()
             .map(|value| compile_resource(&ctx, ctx.as_resource_ref(value)))
+            .map(|res| res.and_then(|schema| intersect_two(schema, siblings.clone())))
             .collect::<Result<Vec<_>>>()?;
         return Ok(Schema::AnyOf { options });
     }
@@ -465,8 +466,9 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
             return Ok(siblings);
         }
         let options = one_of
-            .iter()
+            .into_iter()
             .map(|value| compile_resource(&ctx, ctx.as_resource_ref(value)))
+            .map(|res| res.and_then(|schema| intersect_two(schema, siblings.clone())))
             .collect::<Result<Vec<_>>>()?;
         return Ok(Schema::OneOf { options });
     }
@@ -496,7 +498,7 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
             }
             let resource = ctx.lookup_resource(&reference)?;
             let resolved_schema = compile_resource(ctx, resource)?;
-            return Ok(merge_two(&siblings, &resolved_schema)?);
+            return Ok(intersect_two(resolved_schema, siblings)?);
         }
     }
 
@@ -740,24 +742,21 @@ fn compile_object(
     })
 }
 
-fn merge(schemas: Vec<&Schema>) -> Result<Schema> {
-    if schemas.is_empty() {
-        bail!("merge called with empty list")
-    }
-    if schemas.iter().all(|schema| matches!(schema, Schema::Any)) {
-        return Ok(Schema::Any);
-    }
-    if let Some(unsat) = schemas
-        .iter()
-        .find(|schema| matches!(schema, Schema::Unsatisfiable { .. }))
-    {
-        // Return the first unsatisfiable schema for debug-ability
-        return Ok((*unsat).to_owned());
+fn intersect(schemas: Vec<Schema>) -> Result<Schema> {
+    let (schemas, unsatisfiable) = schemas
+        .into_iter()
+        // "Any" schemas can be ignored
+        .filter(|schema| !matches!(schema, Schema::Any))
+        // Split into unsatisfiable and satisfiable schemas
+        .partition::<Vec<_>, _>(|schema| !matches!(schema, Schema::Unsatisfiable { .. }));
+
+    if let Some(schema) = unsatisfiable.into_iter().next() {
+        return Ok(schema);
     }
-    // TODO: can we avoid cloning here?
-    let mut merged = schemas[0].to_owned();
-    for subschema in &schemas[1..] {
-        merged = merge_two(&merged, subschema)?;
+
+    let mut merged = Schema::Any;
+    for subschema in schemas.into_iter() {
+        merged = intersect_two(merged, subschema)?;
         if matches!(merged, Schema::Unsatisfiable { .. }) {
             // Early exit if the schema is already unsatisfiable
             break;
@@ -766,36 +765,36 @@ fn merge(schemas: Vec<&Schema>) -> Result<Schema> {
     Ok(merged)
 }
 
-fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
+fn intersect_two(schema0: Schema, schema1: Schema) -> Result<Schema> {
     match (schema0, schema1) {
-        (Schema::Any, _) => Ok(schema1.to_owned()),
-        (_, Schema::Any) => Ok(schema0.to_owned()),
-        (Schema::Unsatisfiable { .. }, _) => Ok(schema0.to_owned()),
-        (_, Schema::Unsatisfiable { .. }) => Ok(schema1.to_owned()),
+        (Schema::Any, schema1) => Ok(schema1),
+        (schema0, Schema::Any) => Ok(schema0),
+        (Schema::Unsatisfiable { reason }, _) => Ok(Schema::Unsatisfiable { reason }),
+        (_, Schema::Unsatisfiable { reason }) => Ok(Schema::Unsatisfiable { reason }),
         (Schema::Ref { .. }, _) => Err(anyhow!("$ref with siblings not implemented")),
         (_, Schema::Ref { .. }) => Err(anyhow!("$ref with siblings not implemented")),
-        (Schema::OneOf { options }, _) => Ok(Schema::OneOf {
+        (Schema::OneOf { options }, schema1) => Ok(Schema::OneOf {
             options: options
-                .iter()
-                .map(|opt| merge_two(opt, schema1))
+                .into_iter()
+                .map(|opt| intersect_two(opt, schema1.clone()))
                 .collect::<Result<Vec<_>>>()?,
         }),
-        (_, Schema::OneOf { options }) => Ok(Schema::OneOf {
+        (schema0, Schema::OneOf { options }) => Ok(Schema::OneOf {
             options: options
-                .iter()
-                .map(|opt| merge_two(schema0, opt))
+                .into_iter()
+                .map(|opt| intersect_two(schema0.clone(), opt))
                 .collect::<Result<Vec<_>>>()?,
         }),
-        (Schema::AnyOf { options }, _) => Ok(Schema::AnyOf {
+        (Schema::AnyOf { options }, schema1) => Ok(Schema::AnyOf {
             options: options
-                .iter()
-                .map(|opt| merge_two(opt, schema1))
+                .into_iter()
+                .map(|opt| intersect_two(opt, schema1.clone()))
                 .collect::<Result<Vec<_>>>()?,
         }),
-        (_, Schema::AnyOf { options }) => Ok(Schema::AnyOf {
+        (schema0, Schema::AnyOf { options }) => Ok(Schema::AnyOf {
             options: options
-                .iter()
-                .map(|opt| merge_two(schema0, opt))
+                .into_iter()
+                .map(|opt| intersect_two(schema0.clone(), opt))
                 .collect::<Result<Vec<_>>>()?,
         }),
         (Schema::Null, Schema::Null) => Ok(Schema::Null),
@@ -816,11 +815,11 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
                 integer: int2,
             },
         ) => Ok(Schema::Number {
-            minimum: opt_max(*min1, *min2),
-            maximum: opt_min(*max1, *max2),
-            exclusive_minimum: opt_max(*emin1, *emin2),
-            exclusive_maximum: opt_min(*emax1, *emax2),
-            integer: *int1 || *int2,
+            minimum: opt_max(min1, min2),
+            maximum: opt_min(max1, max2),
+            exclusive_minimum: opt_max(emin1, emin2),
+            exclusive_maximum: opt_min(emax1, emax2),
+            integer: int1 || int2,
         }),
         (
             Schema::String {
@@ -836,15 +835,15 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
                 format: format2,
             },
         ) => Ok(Schema::String {
-            min_length: *min1.max(min2),
-            max_length: opt_min(*max1, *max2),
+            min_length: min1.max(min2),
+            max_length: opt_min(max1, max2),
             pattern: match (pattern1, pattern2) {
                 (None, None) => None,
-                (None, Some(r)) => Some(r.clone()),
-                (Some(r), None) => Some(r.clone()),
+                (None, Some(r)) => Some(r),
+                (Some(r), None) => Some(r),
                 (Some(r1), Some(r2)) => {
                     if r1 == r2 {
-                        Some(r1.clone())
+                        Some(r1)
                     } else {
                         bail!("intersection of patterns not implemented")
                     }
@@ -852,11 +851,11 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
             },
             format: match (format1, format2) {
                 (None, None) => None,
-                (None, Some(fmt)) => Some(fmt.clone()),
-                (Some(fmt), None) => Some(fmt.clone()),
+                (None, Some(fmt)) => Some(fmt),
+                (Some(fmt), None) => Some(fmt),
                 (Some(fmt1), Some(fmt2)) => {
                     if fmt1 == fmt2 {
-                        Some(fmt1.clone())
+                        Some(fmt1)
                     } else {
                         bail!("intersection of formats not implemented")
                     }
@@ -867,32 +866,33 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
             Schema::Array {
                 min_items: min1,
                 max_items: max1,
-                prefix_items: prefix1,
+                prefix_items: mut prefix1,
                 items: items1,
             },
             Schema::Array {
                 min_items: min2,
                 max_items: max2,
-                prefix_items: prefix2,
+                prefix_items: mut prefix2,
                 items: items2,
             },
         ) => Ok(Schema::Array {
-            min_items: *min1.max(min2),
-            max_items: opt_min(*max1, *max2),
-            prefix_items: zip_default(
-                prefix1,
-                prefix2,
-                items2.as_deref().unwrap_or(&Schema::Any),
-                items1.as_deref().unwrap_or(&Schema::Any),
-            )
-            .iter()
-            .map(|(item1, item2)| merge_two(item1, item2))
-            .collect::<Result<Vec<Schema>>>()?,
+            min_items: min1.max(min2),
+            max_items: opt_min(max1, max2),
+            prefix_items: {
+                let len = prefix1.len().max(prefix2.len());
+                prefix1.resize(len, items2.as_deref().cloned().unwrap_or(Schema::Any));
+                prefix2.resize(len, items1.as_deref().cloned().unwrap_or(Schema::Any));
+                prefix1
+                    .into_iter()
+                    .zip(prefix2.into_iter())
+                    .map(|(item1, item2)| intersect_two(item1, item2))
+                    .collect::<Result<Vec<_>>>()?
+            },
             items: match (items1, items2) {
                 (None, None) => None,
-                (None, Some(item)) => Some(Box::new(*item.clone())),
-                (Some(item), None) => Some(Box::new(*item.clone())),
-                (Some(item1), Some(item2)) => Some(Box::new(merge_two(&item1, &item2)?)),
+                (None, Some(item)) => Some(item),
+                (Some(item), None) => Some(item),
+                (Some(item1), Some(item2)) => Some(Box::new(intersect_two(*item1, *item2)?)),
             },
         }),
         (
@@ -902,32 +902,34 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
                 required: req1,
             },
             Schema::Object {
-                properties: props2,
+                properties: mut props2,
                 additional_properties: add2,
                 required: req2,
             },
         ) => {
             let mut new_props = IndexMap::new();
-            for key in props1.keys().chain(props2.keys()) {
-                let new_schema = match (props1.get(key), props2.get(key), add1, add2) {
-                    (Some(schema1), Some(schema2), _, _) => merge_two(schema1, schema2)?,
-                    (Some(schema1), None, _, Some(add)) => merge_two(schema1, &add)?,
-                    (None, Some(schema2), Some(add), _) => merge_two(&add, schema2)?,
-                    (Some(schema1), None, _, None) => schema1.to_owned(),
-                    (None, Some(schema2), None, _) => schema2.to_owned(),
-                    (None, None, _, _) => bail!("should not happen"),
-                };
-                new_props.insert(key.clone(), new_schema);
+            for (key, prop1) in props1.into_iter() {
+                let prop2 = props2
+                    .shift_remove(&key)
+                    .or_else(|| add2.as_deref().cloned())
+                    .unwrap_or(Schema::Any);
+                new_props.insert(key, intersect_two(prop1, prop2)?);
             }
+            for (key, prop2) in props2.into_iter() {
+                let prop1 = add1.as_deref().cloned().unwrap_or(Schema::Any);
+                new_props.insert(key, intersect_two(prop1, prop2)?);
+            }
+            let mut required = req1;
+            required.extend(req2);
             Ok(Schema::Object {
                 properties: new_props,
                 additional_properties: match (add1, add2) {
                     (None, None) => None,
-                    (None, Some(add)) => Some(Box::new(*add.clone())),
-                    (Some(add), None) => Some(Box::new(*add.clone())),
-                    (Some(add1), Some(add2)) => Some(Box::new(merge_two(&add1, &add2)?)),
+                    (None, Some(add2)) => Some(add2),
+                    (Some(add1), None) => Some(add1),
+                    (Some(add1), Some(add2)) => Some(Box::new(intersect_two(*add1, *add2)?)),
                 },
-                required: req1.union(req2).cloned().collect(),
+                required,
             })
         }
         //TODO: get types for error message
@@ -937,17 +939,6 @@ fn merge_two(schema0: &Schema, schema1: &Schema) -> Result<Schema> {
     }
 }
 
-fn zip_default<'a, T>(
-    arr1: &'a [T],
-    arr2: &'a [T],
-    default1: &'a T,
-    default2: &'a T,
-) -> Vec<(&'a T, &'a T)> {
-    let iter1 = arr1.iter().chain(std::iter::repeat(default1));
-    let iter2 = arr2.iter().chain(std::iter::repeat(default2));
-    iter1.zip(iter2).take(arr1.len().max(arr2.len())).collect()
-}
-
 fn opt_max<T: PartialOrd>(a: Option<T>, b: Option<T>) -> Option<T> {
     match (a, b) {
         (Some(a), Some(b)) => {

From 31c3ba03bb7a49e655db3cc06728f068a29ec707 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 19 Nov 2024 09:05:02 -0800
Subject: [PATCH 61/73] mark seen

---
 parser/src/json/schema.rs | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index 257ad90..ca12033 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -1,4 +1,8 @@
-use std::{cell::RefCell, collections::HashMap, rc::Rc};
+use std::{
+    cell::RefCell,
+    collections::{HashMap, HashSet},
+    rc::Rc,
+};
 
 use anyhow::{anyhow, bail, Result};
 use indexmap::{IndexMap, IndexSet};
@@ -247,6 +251,7 @@ struct Context<'a> {
     // TODO: actually use this to handle draft-specific behavior
     draft: Draft,
     defs: Rc<RefCell<HashMap<String, Schema>>>,
+    seen: Rc<RefCell<HashSet<String>>>,
 }
 
 impl<'a> Context<'a> {
@@ -256,6 +261,7 @@ impl<'a> Context<'a> {
             resolver: resolver,
             draft: resource.draft(),
             defs: Rc::clone(&self.defs),
+            seen: Rc::clone(&self.seen),
         })
     }
 
@@ -282,8 +288,12 @@ impl<'a> Context<'a> {
         self.defs.borrow_mut().insert(uri.to_string(), schema);
     }
 
-    fn contains_ref(&self, uri: &str) -> bool {
-        self.defs.borrow().contains_key(uri)
+    fn mark_seen(&self, uri: &str) {
+        self.seen.borrow_mut().insert(uri.to_string());
+    }
+
+    fn been_seen(&self, uri: &str) -> bool {
+        self.seen.borrow().contains(uri)
     }
 
     fn is_valid_keyword(&self, keyword: &str) -> bool {
@@ -330,6 +340,7 @@ pub fn build_schema(contents: &Value) -> Result<(Schema, HashMap<String, Schema>
         resolver: resolver,
         draft: draft,
         defs: Rc::new(RefCell::new(HashMap::new())),
+        seen: Rc::new(RefCell::new(HashSet::new())),
     };
 
     let schema = compile_resource(&ctx, resource_ref)?;
@@ -482,23 +493,20 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
         let uri: String = ctx.normalize_ref(&reference)?.into_string();
         let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
         if siblings == Schema::Any {
-            let placeholder = Schema::Ref { uri: uri.clone() };
-            if ctx.contains_ref(uri.as_str()) {
-                // We've already built this ref, so we can just return it
-                return Ok(placeholder);
+            if !ctx.been_seen(&uri) {
+                ctx.mark_seen(&uri);
+                let resource = ctx.lookup_resource(&reference)?;
+                let resolved_schema = compile_resource(ctx, resource)?;
+                ctx.insert_ref(&uri, resolved_schema);
             }
-            ctx.insert_ref(uri.as_str(), placeholder.clone());
-            let resource = ctx.lookup_resource(&reference)?;
-            let resolved_schema = compile_resource(ctx, resource)?;
-            ctx.insert_ref(uri.as_str(), resolved_schema.clone());
-            return Ok(placeholder);
+            return Ok(Schema::Ref { uri });
         } else {
-            if ctx.contains_ref(uri.as_str()) {
+            if ctx.been_seen(&uri) {
                 bail!("Recursive $refs with sibling keys not implemented")
             }
             let resource = ctx.lookup_resource(&reference)?;
             let resolved_schema = compile_resource(ctx, resource)?;
-            return Ok(intersect_two(resolved_schema, siblings)?);
+            return Ok(intersect_two(siblings, resolved_schema)?);
         }
     }
 

From ee592b0f195495b6d8edc2c62a40e510a827af60 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 19 Nov 2024 09:52:52 -0800
Subject: [PATCH 62/73] make normalize shallow so we don't recursively
 re-normalize schemas that have already been normalized

---
 parser/src/json/schema.rs | 184 +++++++++++++++-----------------------
 1 file changed, 74 insertions(+), 110 deletions(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index ca12033..edda973 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -127,121 +127,83 @@ pub enum Schema {
 impl Schema {
     pub fn false_schema() -> Schema {
         Schema::Unsatisfiable {
-            reason: "Schema is false".to_string(),
+            reason: "schema is false".to_string(),
         }
     }
 
-    fn normalize(self) -> Result<Schema> {
+    /// Shallowly normalize the schema, removing any unnecessary nesting or empty options.
+    fn normalize(self) -> Schema {
         match self {
-            Schema::Any => Ok(self),
-            Schema::Unsatisfiable { .. } => Ok(self),
-            Schema::Null => Ok(self),
-            Schema::Boolean => Ok(self),
-            Schema::Number { .. } => {
-                // TODO: validation logic, maybe returning Schema::Unsatisfiable
-                Ok(self)
-            }
-            Schema::String { .. } => {
-                // TODO: validation logic, maybe returning Schema::Unsatisfiable
-                Ok(self)
-            }
-            Schema::Array {
-                min_items,
-                max_items,
-                prefix_items,
-                items,
-            } => {
-                // TODO: validation logic, maybe returning Schema::Unsatisfiable
-                Ok(Schema::Array {
-                    min_items,
-                    max_items,
-                    prefix_items: prefix_items
-                        .into_iter()
-                        .map(|v| v.normalize())
-                        .collect::<Result<Vec<_>>>()?,
-                    items: items.map(|v| v.normalize().map(Box::new)).transpose()?,
-                })
-            }
-            Schema::Object {
-                properties,
-                additional_properties,
-                required,
-            } => {
-                // TODO: validation logic, maybe returning Schema::Unsatisfiable
-                Ok(Schema::Object {
-                    properties: properties
-                        .into_iter()
-                        .map(|(k, v)| v.normalize().map(|v| (k, v)))
-                        .collect::<Result<IndexMap<_, _>>>()?,
-                    additional_properties: additional_properties
-                        .map(|v| v.normalize().map(Box::new))
-                        .transpose()?,
-                    required: required,
-                })
-            }
-            Schema::Const { .. } => Ok(self),
             Schema::Enum { options } => {
                 if options.is_empty() {
-                    return Ok(Schema::Unsatisfiable {
+                    Schema::Unsatisfiable {
                         reason: "enum is empty".to_string(),
-                    });
+                    }
+                } else {
+                    Schema::Enum { options }
                 }
-                Ok(Schema::Enum { options })
             }
             Schema::AnyOf { options } => {
-                if options.is_empty() {
-                    return Ok(Schema::Unsatisfiable {
-                        reason: "anyOf is empty".to_string(),
-                    });
-                }
                 let mut unsats = Vec::new();
                 let mut valid = Vec::new();
-                for option in options {
-                    let normed = option.normalize()?;
-                    match normed {
-                        Schema::Unsatisfiable { .. } => unsats.push(normed),
-                        // Flatten nested anyOfs
+                for option in options.into_iter() {
+                    match option {
+                        Schema::Any => {
+                            return Schema::Any;
+                        }
+                        Schema::Unsatisfiable { reason } => {
+                            unsats.push(Schema::Unsatisfiable { reason })
+                        }
                         Schema::AnyOf { options: nested } => valid.extend(nested),
-                        _ => valid.push(normed),
+                        other => valid.push(other),
                     }
                 }
                 if valid.is_empty() {
                     // Return the first unsatisfiable schema for debug-ability
-                    return Ok(unsats.swap_remove(0));
+                    if let Some(unsat) = unsats.into_iter().next() {
+                        return unsat;
+                    }
+                    // We must not have had any schemas to begin with
+                    return Schema::Unsatisfiable {
+                        reason: "anyOf is empty".to_string(),
+                    };
                 }
                 if valid.len() == 1 {
-                    return Ok(valid.swap_remove(0));
+                    // Unwrap singleton
+                    return valid.swap_remove(0);
                 }
-                Ok(Schema::AnyOf { options: valid })
+                Schema::AnyOf { options: valid }
             }
             Schema::OneOf { options } => {
-                if options.is_empty() {
-                    return Ok(Schema::Unsatisfiable {
-                        reason: "oneOf is empty".to_string(),
-                    });
-                }
                 let mut unsats = Vec::new();
                 let mut valid = Vec::new();
-                for option in options {
-                    let normed = option.normalize()?;
-                    match normed {
-                        Schema::Unsatisfiable { .. } => unsats.push(normed),
+                for option in options.into_iter() {
+                    match option {
+                        Schema::Unsatisfiable { reason } => {
+                            unsats.push(Schema::Unsatisfiable { reason })
+                        }
                         // Flatten nested oneOfs: (A⊕B)⊕(C⊕D) = A⊕B⊕C⊕D
                         Schema::OneOf { options: nested } => valid.extend(nested),
-                        _ => valid.push(normed),
+                        other => valid.push(other),
                     }
                 }
                 if valid.is_empty() {
                     // Return the first unsatisfiable schema for debug-ability
-                    return Ok(unsats.swap_remove(0));
+                    if let Some(unsat) = unsats.into_iter().next() {
+                        return unsat;
+                    }
+                    // We must not have had any schemas to begin with
+                    return Schema::Unsatisfiable {
+                        reason: "oneOf is empty".to_string(),
+                    };
                 }
                 if valid.len() == 1 {
-                    return Ok(valid.swap_remove(0));
+                    // Unwrap singleton
+                    return valid.swap_remove(0);
                 }
-                Ok(Schema::OneOf { options: valid })
+                Schema::OneOf { options: valid }
             }
-            // TODO: ?
-            Schema::Ref { .. } => Ok(self),
+            other_schema => other_schema,
         }
     }
 }
@@ -354,7 +316,7 @@ fn compile_resource(ctx: &Context, resource: ResourceRef) -> Result<Schema> {
 }
 
 fn compile_contents(ctx: &Context, contents: &Value) -> Result<Schema> {
-    compile_contents_inner(ctx, contents)?.normalize()
+    compile_contents_inner(ctx, contents).map(|schema| schema.normalize())
 }
 
 fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
@@ -773,40 +735,41 @@ fn intersect(schemas: Vec<Schema>) -> Result<Schema> {
     Ok(merged)
 }
 
+/// Intersect two schemas, returning a new (normalized) schema that represents the intersection of the two.
 fn intersect_two(schema0: Schema, schema1: Schema) -> Result<Schema> {
-    match (schema0, schema1) {
-        (Schema::Any, schema1) => Ok(schema1),
-        (schema0, Schema::Any) => Ok(schema0),
-        (Schema::Unsatisfiable { reason }, _) => Ok(Schema::Unsatisfiable { reason }),
-        (_, Schema::Unsatisfiable { reason }) => Ok(Schema::Unsatisfiable { reason }),
-        (Schema::Ref { .. }, _) => Err(anyhow!("$ref with siblings not implemented")),
-        (_, Schema::Ref { .. }) => Err(anyhow!("$ref with siblings not implemented")),
-        (Schema::OneOf { options }, schema1) => Ok(Schema::OneOf {
+    let merged = match (schema0, schema1) {
+        (Schema::Any, schema1) => schema1,
+        (schema0, Schema::Any) => schema0,
+        (Schema::Unsatisfiable { reason }, _) => Schema::Unsatisfiable { reason },
+        (_, Schema::Unsatisfiable { reason }) => Schema::Unsatisfiable { reason },
+        (Schema::Ref { .. }, _) => bail!("$ref with siblings not implemented"),
+        (_, Schema::Ref { .. }) => bail!("$ref with siblings not implemented"),
+        (Schema::OneOf { options }, schema1) => Schema::OneOf {
             options: options
                 .into_iter()
                 .map(|opt| intersect_two(opt, schema1.clone()))
                 .collect::<Result<Vec<_>>>()?,
-        }),
-        (schema0, Schema::OneOf { options }) => Ok(Schema::OneOf {
+        },
+        (schema0, Schema::OneOf { options }) => Schema::OneOf {
             options: options
                 .into_iter()
                 .map(|opt| intersect_two(schema0.clone(), opt))
                 .collect::<Result<Vec<_>>>()?,
-        }),
-        (Schema::AnyOf { options }, schema1) => Ok(Schema::AnyOf {
+        },
+        (Schema::AnyOf { options }, schema1) => Schema::AnyOf {
             options: options
                 .into_iter()
                 .map(|opt| intersect_two(opt, schema1.clone()))
                 .collect::<Result<Vec<_>>>()?,
-        }),
-        (schema0, Schema::AnyOf { options }) => Ok(Schema::AnyOf {
+        },
+        (schema0, Schema::AnyOf { options }) => Schema::AnyOf {
             options: options
                 .into_iter()
                 .map(|opt| intersect_two(schema0.clone(), opt))
                 .collect::<Result<Vec<_>>>()?,
-        }),
-        (Schema::Null, Schema::Null) => Ok(Schema::Null),
-        (Schema::Boolean, Schema::Boolean) => Ok(Schema::Boolean),
+        },
+        (Schema::Null, Schema::Null) => Schema::Null,
+        (Schema::Boolean, Schema::Boolean) => Schema::Boolean,
         (
             Schema::Number {
                 minimum: min1,
@@ -822,13 +785,13 @@ fn intersect_two(schema0: Schema, schema1: Schema) -> Result<Schema> {
                 exclusive_maximum: emax2,
                 integer: int2,
             },
-        ) => Ok(Schema::Number {
+        ) => Schema::Number {
             minimum: opt_max(min1, min2),
             maximum: opt_min(max1, max2),
             exclusive_minimum: opt_max(emin1, emin2),
             exclusive_maximum: opt_min(emax1, emax2),
             integer: int1 || int2,
-        }),
+        },
         (
             Schema::String {
                 min_length: min1,
@@ -842,7 +805,7 @@ fn intersect_two(schema0: Schema, schema1: Schema) -> Result<Schema> {
                 pattern: pattern2,
                 format: format2,
             },
-        ) => Ok(Schema::String {
+        ) => Schema::String {
             min_length: min1.max(min2),
             max_length: opt_min(max1, max2),
             pattern: match (pattern1, pattern2) {
@@ -869,7 +832,7 @@ fn intersect_two(schema0: Schema, schema1: Schema) -> Result<Schema> {
                     }
                 }
             },
-        }),
+        },
         (
             Schema::Array {
                 min_items: min1,
@@ -883,7 +846,7 @@ fn intersect_two(schema0: Schema, schema1: Schema) -> Result<Schema> {
                 prefix_items: mut prefix2,
                 items: items2,
             },
-        ) => Ok(Schema::Array {
+        ) => Schema::Array {
             min_items: min1.max(min2),
             max_items: opt_min(max1, max2),
             prefix_items: {
@@ -902,7 +865,7 @@ fn intersect_two(schema0: Schema, schema1: Schema) -> Result<Schema> {
                 (Some(item), None) => Some(item),
                 (Some(item1), Some(item2)) => Some(Box::new(intersect_two(*item1, *item2)?)),
             },
-        }),
+        },
         (
             Schema::Object {
                 properties: props1,
@@ -929,7 +892,7 @@ fn intersect_two(schema0: Schema, schema1: Schema) -> Result<Schema> {
             }
             let mut required = req1;
             required.extend(req2);
-            Ok(Schema::Object {
+            Schema::Object {
                 properties: new_props,
                 additional_properties: match (add1, add2) {
                     (None, None) => None,
@@ -938,13 +901,14 @@ fn intersect_two(schema0: Schema, schema1: Schema) -> Result<Schema> {
                     (Some(add1), Some(add2)) => Some(Box::new(intersect_two(*add1, *add2)?)),
                 },
                 required,
-            })
+            }
         }
         //TODO: get types for error message
-        _ => Ok(Schema::Unsatisfiable {
+        _ => Schema::Unsatisfiable {
             reason: "incompatible types".to_string(),
-        }),
-    }
+        },
+    };
+    Ok(merged.normalize())
 }
 
 fn opt_max<T: PartialOrd>(a: Option<T>, b: Option<T>) -> Option<T> {

From fd2e16665a99d0e09185a09af83485a8ca5b33ee Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 19 Nov 2024 12:53:37 -0800
Subject: [PATCH 63/73] fix all but the most degenerate refs-with-sibling-keys

---
 parser/src/json/schema.rs | 141 ++++++++++++++++++++++++++++++--------
 1 file changed, 112 insertions(+), 29 deletions(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index edda973..dc460b9 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -250,6 +250,10 @@ impl<'a> Context<'a> {
         self.defs.borrow_mut().insert(uri.to_string(), schema);
     }
 
+    fn get_ref_cloned(&self, uri: &str) -> Option<Schema> {
+        self.defs.borrow().get(uri).cloned()
+    }
+
     fn mark_seen(&self, uri: &str) {
         self.seen.borrow_mut().insert(uri.to_string());
     }
@@ -407,7 +411,7 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
             .iter()
             .map(|value| compile_resource(&ctx, ctx.as_resource_ref(value)))
             .collect::<Result<Vec<_>>>()?;
-        let merged = intersect(options.into_iter().chain(vec![siblings]).collect())?;
+        let merged = intersect(ctx, options.into_iter().chain(vec![siblings]).collect())?;
         return Ok(merged);
     }
 
@@ -423,7 +427,7 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
         let options = any_of
             .into_iter()
             .map(|value| compile_resource(&ctx, ctx.as_resource_ref(value)))
-            .map(|res| res.and_then(|schema| intersect_two(schema, siblings.clone())))
+            .map(|res| res.and_then(|schema| intersect_two(ctx, schema, siblings.clone())))
             .collect::<Result<Vec<_>>>()?;
         return Ok(Schema::AnyOf { options });
     }
@@ -441,7 +445,7 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
         let options = one_of
             .into_iter()
             .map(|value| compile_resource(&ctx, ctx.as_resource_ref(value)))
-            .map(|res| res.and_then(|schema| intersect_two(schema, siblings.clone())))
+            .map(|res| res.and_then(|schema| intersect_two(ctx, schema, siblings.clone())))
             .collect::<Result<Vec<_>>>()?;
         return Ok(Schema::OneOf { options });
     }
@@ -455,20 +459,10 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
         let uri: String = ctx.normalize_ref(&reference)?.into_string();
         let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
         if siblings == Schema::Any {
-            if !ctx.been_seen(&uri) {
-                ctx.mark_seen(&uri);
-                let resource = ctx.lookup_resource(&reference)?;
-                let resolved_schema = compile_resource(ctx, resource)?;
-                ctx.insert_ref(&uri, resolved_schema);
-            }
+            define_ref(ctx, &uri)?;
             return Ok(Schema::Ref { uri });
         } else {
-            if ctx.been_seen(&uri) {
-                bail!("Recursive $refs with sibling keys not implemented")
-            }
-            let resource = ctx.lookup_resource(&reference)?;
-            let resolved_schema = compile_resource(ctx, resource)?;
-            return Ok(intersect_two(siblings, resolved_schema)?);
+            return intersect_ref(ctx, &uri, siblings);
         }
     }
 
@@ -495,6 +489,33 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
     Ok(Schema::AnyOf { options })
 }
 
+fn define_ref(ctx: &Context, ref_uri: &str) -> Result<()> {
+    if !ctx.been_seen(ref_uri) {
+        ctx.mark_seen(ref_uri);
+        let resource = ctx.lookup_resource(ref_uri)?;
+        let resolved_schema = compile_resource(ctx, resource)?;
+        ctx.insert_ref(&ref_uri, resolved_schema);
+    }
+    Ok(())
+}
+
+fn intersect_ref(ctx: &Context, ref_uri: &str, schema: Schema) -> Result<Schema> {
+    define_ref(ctx, ref_uri)?;
+    let resolved_schema = ctx
+        .get_ref_cloned(ref_uri)
+        // The ref might not have been defined if we're in a recursive loop and every ref in the loop
+        // has a sibling key.
+        // TODO: add an extra layer of indirection by defining a URI for the current location (e.g. by hashing the serialized sibling schema)
+        // and returning a ref to that URI here to break the loop.
+        .ok_or_else(|| {
+            anyhow!(
+                "circular references with sibling keys are not supported: {}",
+                ref_uri
+            )
+        })?;
+    intersect_two(ctx, schema, resolved_schema)
+}
+
 fn compile_type(ctx: &Context, tp: &str, schema: &Map<String, Value>) -> Result<Schema> {
     match tp {
         "null" => Ok(Schema::Null),
@@ -712,7 +733,7 @@ fn compile_object(
     })
 }
 
-fn intersect(schemas: Vec<Schema>) -> Result<Schema> {
+fn intersect(ctx: &Context, schemas: Vec<Schema>) -> Result<Schema> {
     let (schemas, unsatisfiable) = schemas
         .into_iter()
         // "Any" schemas can be ignored
@@ -726,7 +747,7 @@ fn intersect(schemas: Vec<Schema>) -> Result<Schema> {
 
     let mut merged = Schema::Any;
     for subschema in schemas.into_iter() {
-        merged = intersect_two(merged, subschema)?;
+        merged = intersect_two(ctx, merged, subschema)?;
         if matches!(merged, Schema::Unsatisfiable { .. }) {
             // Early exit if the schema is already unsatisfiable
             break;
@@ -736,36 +757,36 @@ fn intersect(schemas: Vec<Schema>) -> Result<Schema> {
 }
 
 /// Intersect two schemas, returning a new (normalized) schema that represents the intersection of the two.
-fn intersect_two(schema0: Schema, schema1: Schema) -> Result<Schema> {
+fn intersect_two(ctx: &Context, schema0: Schema, schema1: Schema) -> Result<Schema> {
     let merged = match (schema0, schema1) {
         (Schema::Any, schema1) => schema1,
         (schema0, Schema::Any) => schema0,
         (Schema::Unsatisfiable { reason }, _) => Schema::Unsatisfiable { reason },
         (_, Schema::Unsatisfiable { reason }) => Schema::Unsatisfiable { reason },
-        (Schema::Ref { .. }, _) => bail!("$ref with siblings not implemented"),
-        (_, Schema::Ref { .. }) => bail!("$ref with siblings not implemented"),
+        (Schema::Ref { uri }, schema1) => intersect_ref(ctx, &uri, schema1)?,
+        (schema0, Schema::Ref { uri }) => intersect_ref(ctx, &uri, schema0)?,
         (Schema::OneOf { options }, schema1) => Schema::OneOf {
             options: options
                 .into_iter()
-                .map(|opt| intersect_two(opt, schema1.clone()))
+                .map(|opt| intersect_two(ctx, opt, schema1.clone()))
                 .collect::<Result<Vec<_>>>()?,
         },
         (schema0, Schema::OneOf { options }) => Schema::OneOf {
             options: options
                 .into_iter()
-                .map(|opt| intersect_two(schema0.clone(), opt))
+                .map(|opt| intersect_two(ctx, schema0.clone(), opt))
                 .collect::<Result<Vec<_>>>()?,
         },
         (Schema::AnyOf { options }, schema1) => Schema::AnyOf {
             options: options
                 .into_iter()
-                .map(|opt| intersect_two(opt, schema1.clone()))
+                .map(|opt| intersect_two(ctx, opt, schema1.clone()))
                 .collect::<Result<Vec<_>>>()?,
         },
         (schema0, Schema::AnyOf { options }) => Schema::AnyOf {
             options: options
                 .into_iter()
-                .map(|opt| intersect_two(schema0.clone(), opt))
+                .map(|opt| intersect_two(ctx, schema0.clone(), opt))
                 .collect::<Result<Vec<_>>>()?,
         },
         (Schema::Null, Schema::Null) => Schema::Null,
@@ -856,14 +877,14 @@ fn intersect_two(schema0: Schema, schema1: Schema) -> Result<Schema> {
                 prefix1
                     .into_iter()
                     .zip(prefix2.into_iter())
-                    .map(|(item1, item2)| intersect_two(item1, item2))
+                    .map(|(item1, item2)| intersect_two(ctx, item1, item2))
                     .collect::<Result<Vec<_>>>()?
             },
             items: match (items1, items2) {
                 (None, None) => None,
                 (None, Some(item)) => Some(item),
                 (Some(item), None) => Some(item),
-                (Some(item1), Some(item2)) => Some(Box::new(intersect_two(*item1, *item2)?)),
+                (Some(item1), Some(item2)) => Some(Box::new(intersect_two(ctx, *item1, *item2)?)),
             },
         },
         (
@@ -884,11 +905,11 @@ fn intersect_two(schema0: Schema, schema1: Schema) -> Result<Schema> {
                     .shift_remove(&key)
                     .or_else(|| add2.as_deref().cloned())
                     .unwrap_or(Schema::Any);
-                new_props.insert(key, intersect_two(prop1, prop2)?);
+                new_props.insert(key, intersect_two(ctx, prop1, prop2)?);
             }
             for (key, prop2) in props2.into_iter() {
                 let prop1 = add1.as_deref().cloned().unwrap_or(Schema::Any);
-                new_props.insert(key, intersect_two(prop1, prop2)?);
+                new_props.insert(key, intersect_two(ctx, prop1, prop2)?);
             }
             let mut required = req1;
             required.extend(req2);
@@ -898,7 +919,7 @@ fn intersect_two(schema0: Schema, schema1: Schema) -> Result<Schema> {
                     (None, None) => None,
                     (None, Some(add2)) => Some(add2),
                     (Some(add1), None) => Some(add1),
-                    (Some(add1), Some(add2)) => Some(Box::new(intersect_two(*add1, *add2)?)),
+                    (Some(add1), Some(add2)) => Some(Box::new(intersect_two(ctx, *add1, *add2)?)),
                 },
                 required,
             }
@@ -1022,4 +1043,66 @@ mod test {
         let schema = build_schema(&schema).unwrap();
         println!("{:?}", schema);
     }
+
+    #[test]
+    fn test_fhir() -> Result<()> {
+        let file = std::fs::read_to_string("../../fhir.schema.json")?;
+        let schema = build_schema(&serde_json::from_str(&file)?)?;
+        println!("{:?}", schema.0);
+        Ok(())
+    }
+
+    #[test]
+    #[ignore]
+    fn test_recursive_ref() {
+        let schema = json!({
+            "type": "object",
+            "properties": {
+                "value": {
+                    "type": "number",
+                },
+                "next": {
+                    "$ref": "#",
+                    "properties": {
+                        "value": {
+                            "type": "integer",
+                        },
+                    }
+                },
+            },
+            "additionalProperties": false,
+            "required": ["value"],
+        });
+        let schema = build_schema(&schema).unwrap();
+        println!("{:?}", schema);
+    }
+
+    #[test]
+    fn test_indirect_recursive_ref() {
+        let schema = json!({
+            "type": "object",
+            "properties": {
+                "value": {
+                    "type": "number",
+                },
+                "next": {
+                    "$ref": "#/$defs/inner",
+                },
+            },
+            "additionalProperties": false,
+            "required": ["value"],
+            "$defs": {
+                "inner": {
+                    "$ref": "#",
+                    "properties": {
+                        "value": {
+                            "type": "integer",
+                        },
+                    }
+                }
+            }
+        });
+        let schema = build_schema(&schema).unwrap();
+        println!("{:?}", schema);
+    }
 }

From 86c2bf3a19521dc9f56ee0718bd5c0dd8f099524 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 19 Nov 2024 12:53:55 -0800
Subject: [PATCH 64/73] cargo fmt

---
 parser/src/json/compiler.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index e30a653..5ac7d03 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -282,7 +282,12 @@ impl Compiler {
                     .into_iter()
                     .map(|v| Schema::Const { value: v })
                     .collect::<Vec<_>>();
-                self.gen_json_array(&prefix_items, &Schema::false_schema(), n_items, Some(n_items))
+                self.gen_json_array(
+                    &prefix_items,
+                    &Schema::false_schema(),
+                    n_items,
+                    Some(n_items),
+                )
             }
             _ => {
                 // let serde_json dump simple values

From 2aa4d876faeff55018f1b5a3146f7ff3084d3b6b Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 19 Nov 2024 13:24:12 -0800
Subject: [PATCH 65/73] encapsulate shared context

---
 parser/src/json/schema.rs | 39 ++++++++++++++++++++++++++-------------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index dc460b9..636a468 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -208,12 +208,24 @@ impl Schema {
     }
 }
 
+struct SharedContext {
+    defs: HashMap<String, Schema>,
+    seen: HashSet<String>,
+}
+
+impl SharedContext {
+    fn new() -> Self {
+        SharedContext {
+            defs: HashMap::new(),
+            seen: HashSet::new(),
+        }
+    }
+}
+
 struct Context<'a> {
     resolver: Resolver<'a>,
-    // TODO: actually use this to handle draft-specific behavior
     draft: Draft,
-    defs: Rc<RefCell<HashMap<String, Schema>>>,
-    seen: Rc<RefCell<HashSet<String>>>,
+    shared: Rc<RefCell<SharedContext>>,
 }
 
 impl<'a> Context<'a> {
@@ -222,8 +234,7 @@ impl<'a> Context<'a> {
         Ok(Context {
             resolver: resolver,
             draft: resource.draft(),
-            defs: Rc::clone(&self.defs),
-            seen: Rc::clone(&self.seen),
+            shared: Rc::clone(&self.shared),
         })
     }
 
@@ -247,19 +258,22 @@ impl<'a> Context<'a> {
     }
 
     fn insert_ref(&self, uri: &str, schema: Schema) {
-        self.defs.borrow_mut().insert(uri.to_string(), schema);
+        self.shared
+            .borrow_mut()
+            .defs
+            .insert(uri.to_string(), schema);
     }
 
     fn get_ref_cloned(&self, uri: &str) -> Option<Schema> {
-        self.defs.borrow().get(uri).cloned()
+        self.shared.borrow().defs.get(uri).cloned()
     }
 
     fn mark_seen(&self, uri: &str) {
-        self.seen.borrow_mut().insert(uri.to_string());
+        self.shared.borrow_mut().seen.insert(uri.to_string());
     }
 
     fn been_seen(&self, uri: &str) -> bool {
-        self.seen.borrow().contains(uri)
+        self.shared.borrow().seen.contains(uri)
     }
 
     fn is_valid_keyword(&self, keyword: &str) -> bool {
@@ -305,13 +319,12 @@ pub fn build_schema(contents: &Value) -> Result<(Schema, HashMap<String, Schema>
     let ctx = Context {
         resolver: resolver,
         draft: draft,
-        defs: Rc::new(RefCell::new(HashMap::new())),
-        seen: Rc::new(RefCell::new(HashSet::new())),
+        shared: Rc::new(RefCell::new(SharedContext::new())),
     };
 
     let schema = compile_resource(&ctx, resource_ref)?;
-    let definitions = ctx.defs.take();
-    Ok((schema, definitions))
+    let defs = std::mem::take(&mut ctx.shared.borrow_mut().defs);
+    Ok((schema, defs))
 }
 
 fn compile_resource(ctx: &Context, resource: ResourceRef) -> Result<Schema> {

From 63507a974f8c5002945d98358beb11cc70db7ce0 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 19 Nov 2024 14:00:14 -0800
Subject: [PATCH 66/73] rough limit on total schema size

---
 parser/src/json/schema.rs | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index 636a468..4c252d4 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -208,9 +208,21 @@ impl Schema {
     }
 }
 
+#[derive(Clone)]
+struct SchemaBuilderOptions {
+    max_size: usize,
+}
+
+impl Default for SchemaBuilderOptions {
+    fn default() -> Self {
+        SchemaBuilderOptions { max_size: 50_000 }
+    }
+}
+
 struct SharedContext {
     defs: HashMap<String, Schema>,
     seen: HashSet<String>,
+    n_compiled: usize,
 }
 
 impl SharedContext {
@@ -218,6 +230,7 @@ impl SharedContext {
         SharedContext {
             defs: HashMap::new(),
             seen: HashSet::new(),
+            n_compiled: 0,
         }
     }
 }
@@ -226,6 +239,7 @@ struct Context<'a> {
     resolver: Resolver<'a>,
     draft: Draft,
     shared: Rc<RefCell<SharedContext>>,
+    options: SchemaBuilderOptions,
 }
 
 impl<'a> Context<'a> {
@@ -235,6 +249,7 @@ impl<'a> Context<'a> {
             resolver: resolver,
             draft: resource.draft(),
             shared: Rc::clone(&self.shared),
+            options: self.options.clone(),
         })
     }
 
@@ -285,6 +300,15 @@ impl<'a> Context<'a> {
         }
         return false;
     }
+
+    fn increment(&self) -> Result<()> {
+        let mut shared = self.shared.borrow_mut();
+        shared.n_compiled += 1;
+        if shared.n_compiled > self.options.max_size {
+            bail!("schema too large");
+        }
+        Ok(())
+    }
 }
 
 fn draft_for(value: &Value) -> Draft {
@@ -320,6 +344,7 @@ pub fn build_schema(contents: &Value) -> Result<(Schema, HashMap<String, Schema>
         resolver: resolver,
         draft: draft,
         shared: Rc::new(RefCell::new(SharedContext::new())),
+        options: SchemaBuilderOptions::default(),
     };
 
     let schema = compile_resource(&ctx, resource_ref)?;
@@ -337,6 +362,8 @@ fn compile_contents(ctx: &Context, contents: &Value) -> Result<Schema> {
 }
 
 fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
+    ctx.increment()?;
+
     if let Some(b) = contents.as_bool() {
         if b {
             return Ok(Schema::Any);
@@ -530,6 +557,8 @@ fn intersect_ref(ctx: &Context, ref_uri: &str, schema: Schema) -> Result<Schema>
 }
 
 fn compile_type(ctx: &Context, tp: &str, schema: &Map<String, Value>) -> Result<Schema> {
+    ctx.increment()?;
+
     match tp {
         "null" => Ok(Schema::Null),
         "boolean" => Ok(Schema::Boolean),
@@ -771,6 +800,8 @@ fn intersect(ctx: &Context, schemas: Vec<Schema>) -> Result<Schema> {
 
 /// Intersect two schemas, returning a new (normalized) schema that represents the intersection of the two.
 fn intersect_two(ctx: &Context, schema0: Schema, schema1: Schema) -> Result<Schema> {
+    ctx.increment()?;
+
     let merged = match (schema0, schema1) {
         (Schema::Any, schema1) => schema1,
         (schema0, Schema::Any) => schema0,

From 5a884e5137a5fa5300366e39b3d9fe06d537d256 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 19 Nov 2024 14:40:23 -0800
Subject: [PATCH 67/73] fix sample_parser JsonCompileOptions

---
 sample_parser/src/minimal.rs       | 4 +---
 sample_parser/src/sample_parser.rs | 4 +---
 sample_parser/src/schema_tester.rs | 2 +-
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/sample_parser/src/minimal.rs b/sample_parser/src/minimal.rs
index a1d4ff6..9e119ae 100644
--- a/sample_parser/src/minimal.rs
+++ b/sample_parser/src/minimal.rs
@@ -56,9 +56,7 @@ fn main() {
     let schema: TopLevelGrammar = if args[1].ends_with(".ll.json") {
         serde_json::from_str(&schema_file).expect("Invalid JSON in schema")
     } else if args[1].ends_with(".schema.json") {
-        let opts = JsonCompileOptions {
-            compact: false,
-        };
+        let opts = JsonCompileOptions::default();
         let val = serde_json::from_str(&schema_file).expect("Invalid JSON in schema");
         opts.json_to_llg(&val)
             .expect("Failed to convert JSON to LLG")
diff --git a/sample_parser/src/sample_parser.rs b/sample_parser/src/sample_parser.rs
index a631073..9bd243e 100644
--- a/sample_parser/src/sample_parser.rs
+++ b/sample_parser/src/sample_parser.rs
@@ -17,9 +17,7 @@ fn main() {
     let schema: TopLevelGrammar = if args[1].ends_with(".ll.json") {
         serde_json::from_str(&schema_file).expect("Invalid JSON in schema")
     } else if args[1].ends_with(".schema.json") {
-        let opts = JsonCompileOptions {
-            compact: false,
-        };
+        let opts = JsonCompileOptions::default();
         let val = serde_json::from_str(&schema_file).expect("Invalid JSON in schema");
         opts.json_to_llg(&val)
             .expect("Failed to convert JSON to LLG")
diff --git a/sample_parser/src/schema_tester.rs b/sample_parser/src/schema_tester.rs
index da7b8e9..5c8736f 100644
--- a/sample_parser/src/schema_tester.rs
+++ b/sample_parser/src/schema_tester.rs
@@ -9,7 +9,7 @@ use serde_json::Value;
 
 fn test_file(tok_env: TokEnv, file: &str) {
     let schema_file = read_file_to_string(file);
-    let opts = JsonCompileOptions { compact: false };
+    let opts = JsonCompileOptions::default();
     let val: Value = serde_json::from_str(&schema_file).expect("Invalid JSON in schema");
 
     if schema_file.len() < 512 && val["$ref"].is_string() {

From a3a184767ef6d327c5960f184858cf28e3518873 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 19 Nov 2024 15:02:55 -0800
Subject: [PATCH 68/73] take Value as value

---
 parser/src/earley/from_guidance.rs |  2 +-
 parser/src/ffi.rs                  |  2 +-
 parser/src/json/compiler.rs        |  8 ++++----
 parser/src/json/schema.rs          | 10 +++++-----
 rust/src/py.rs                     |  2 +-
 sample_parser/src/minimal.rs       |  2 +-
 sample_parser/src/sample_parser.rs |  2 +-
 sample_parser/src/schema_tester.rs |  2 +-
 8 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/parser/src/earley/from_guidance.rs b/parser/src/earley/from_guidance.rs
index 65d6c82..a7600a0 100644
--- a/parser/src/earley/from_guidance.rs
+++ b/parser/src/earley/from_guidance.rs
@@ -93,7 +93,7 @@ fn grammar_from_json(
             "cannot have both json_schema/lark_grammar and nodes/rx_nodes"
         );
 
-        let mut new_grm = if let Some(json_schema) = input.json_schema.as_ref() {
+        let mut new_grm = if let Some(json_schema) = input.json_schema.take() {
             ensure!(
                 input.lark_grammar.is_none(),
                 "cannot have both json_schema and lark_grammar"
diff --git a/parser/src/ffi.rs b/parser/src/ffi.rs
index 22b9fc2..1c752b9 100644
--- a/parser/src/ffi.rs
+++ b/parser/src/ffi.rs
@@ -358,7 +358,7 @@ fn new_constraint_json(init: &LlgConstraintInit, json_schema: *const c_char) ->
         .map_err(|e| anyhow::anyhow!("Invalid JSON in json_schema: {e}"))?;
     let opts = JsonCompileOptions::default();
     let grammar = opts
-        .json_to_llg(&json_schema)
+        .json_to_llg(json_schema)
         .map_err(|e| anyhow::anyhow!("Error compiling JSON schema to LLG: {e}"))?;
     init.build_constraint(grammar)
 }
diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index 5ac7d03..e614941 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -103,19 +103,19 @@ impl Default for JsonCompileOptions {
 }
 
 impl JsonCompileOptions {
-    pub fn json_to_llg(&self, schema: &Value) -> Result<TopLevelGrammar> {
+    pub fn json_to_llg(&self, schema: Value) -> Result<TopLevelGrammar> {
         let mut compiler = Compiler::new(self.clone());
         #[cfg(feature = "jsonschema_validation")]
         {
             use crate::json_validation::validate_schema;
-            validate_schema(schema)?;
+            validate_schema(&schema)?;
         }
 
         compiler.execute(schema)?;
         compiler.builder.finalize()
     }
 
-    pub fn json_to_llg_no_validate(&self, schema: &Value) -> Result<TopLevelGrammar> {
+    pub fn json_to_llg_no_validate(&self, schema: Value) -> Result<TopLevelGrammar> {
         let mut compiler = Compiler::new(self.clone());
         compiler.execute(schema)?;
         compiler.builder.finalize()
@@ -138,7 +138,7 @@ impl Compiler {
         }
     }
 
-    pub fn execute(&mut self, schema: &Value) -> Result<()> {
+    pub fn execute(&mut self, schema: Value) -> Result<()> {
         self.builder.add_grammar(GrammarWithLexer {
             greedy_skip_rx: if self.options.whitespace_flexible {
                 Some(mk_regex(r"[\x20\x0A\x0D\x09]+"))
diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index 4c252d4..f8d6bf1 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -323,7 +323,7 @@ fn instance_if_valid<'a>(instance: &'a Value, validator: &'a Validator) -> Optio
     }
 }
 
-pub fn build_schema(contents: &Value) -> Result<(Schema, HashMap<String, Schema>)> {
+pub fn build_schema(contents: Value) -> Result<(Schema, HashMap<String, Schema>)> {
     if let Some(b) = contents.as_bool() {
         if b {
             return Ok((Schema::Any, HashMap::new()));
@@ -332,9 +332,8 @@ pub fn build_schema(contents: &Value) -> Result<(Schema, HashMap<String, Schema>
         }
     }
 
-    let draft = draft_for(contents);
-    let resource_ref = draft.create_resource_ref(contents);
-    let resource = draft.create_resource(contents.clone());
+    let draft = draft_for(&contents);
+    let resource = draft.create_resource(contents);
     let base_uri = resource.id().unwrap_or(DEFAULT_ROOT_URI).to_string();
 
     let registry = Registry::try_new(&base_uri, resource)?;
@@ -347,7 +346,8 @@ pub fn build_schema(contents: &Value) -> Result<(Schema, HashMap<String, Schema>
         options: SchemaBuilderOptions::default(),
     };
 
-    let schema = compile_resource(&ctx, resource_ref)?;
+    let root_resource = ctx.lookup_resource(&base_uri)?;
+    let schema = compile_resource(&ctx, root_resource)?;
     let defs = std::mem::take(&mut ctx.shared.borrow_mut().defs);
     Ok((schema, defs))
 }
diff --git a/rust/src/py.rs b/rust/src/py.rs
index 4abf1a8..11fd191 100644
--- a/rust/src/py.rs
+++ b/rust/src/py.rs
@@ -272,7 +272,7 @@ impl JsonCompiler {
             key_separator: self.key_separator.clone(),
             whitespace_flexible: self.whitespace_flexible,
         };
-        let tlg = compile_options.json_to_llg(&schema).map_err(val_error)?;
+        let tlg = compile_options.json_to_llg(schema).map_err(val_error)?;
         let grammar = &tlg.grammars[0];
         Ok(serde_json::to_string(grammar).map_err(val_error)?)
     }
diff --git a/sample_parser/src/minimal.rs b/sample_parser/src/minimal.rs
index 9e119ae..112e0a2 100644
--- a/sample_parser/src/minimal.rs
+++ b/sample_parser/src/minimal.rs
@@ -58,7 +58,7 @@ fn main() {
     } else if args[1].ends_with(".schema.json") {
         let opts = JsonCompileOptions::default();
         let val = serde_json::from_str(&schema_file).expect("Invalid JSON in schema");
-        opts.json_to_llg(&val)
+        opts.json_to_llg(val)
             .expect("Failed to convert JSON to LLG")
     } else {
         panic!("Unknown schema file extension")
diff --git a/sample_parser/src/sample_parser.rs b/sample_parser/src/sample_parser.rs
index 302d106..4022156 100644
--- a/sample_parser/src/sample_parser.rs
+++ b/sample_parser/src/sample_parser.rs
@@ -20,7 +20,7 @@ fn main() {
     } else if args[1].ends_with(".schema.json") {
         let opts = JsonCompileOptions::default();
         let val = serde_json::from_str(&schema_file).expect("Invalid JSON in schema");
-        opts.json_to_llg(&val)
+        opts.json_to_llg(val)
             .expect("Failed to convert JSON to LLG")
     } else if args[1].ends_with(".lark") {
         lark_to_llguidance(&schema_file).expect("Failed to convert lark to LLG")
diff --git a/sample_parser/src/schema_tester.rs b/sample_parser/src/schema_tester.rs
index 5c8736f..154d0c3 100644
--- a/sample_parser/src/schema_tester.rs
+++ b/sample_parser/src/schema_tester.rs
@@ -17,7 +17,7 @@ fn test_file(tok_env: TokEnv, file: &str) {
         return;
     }
 
-    let schema = opts.json_to_llg(&val);
+    let schema = opts.json_to_llg(val);
 
     let schema = match schema {
         Ok(schema) => schema,

From 6b9762f8a97b11ff5d8629eca0e87b601003c18b Mon Sep 17 00:00:00 2001
From: Michal Moskal <michal@moskal.me>
Date: Tue, 19 Nov 2024 15:09:31 -0800
Subject: [PATCH 69/73] use shallow clone

---
 parser/src/json/schema.rs | 76 +++++++++++++++++++++++----------------
 1 file changed, 45 insertions(+), 31 deletions(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index 4c252d4..060879a 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -362,8 +362,6 @@ fn compile_contents(ctx: &Context, contents: &Value) -> Result<Schema> {
 }
 
 fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
-    ctx.increment()?;
-
     if let Some(b) = contents.as_bool() {
         if b {
             return Ok(Schema::Any);
@@ -378,11 +376,28 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
         .as_object()
         .ok_or_else(|| anyhow!("schema must be an object or boolean"))?;
 
+    // Make a mutable copy of the schema so we can modify it
+    let schemadict = schemadict
+        .iter()
+        .map(|(k, v)| (k.as_str(), v))
+        .collect::<HashMap<_, _>>();
+
+    compile_contents_map(ctx, schemadict)
+}
+
+fn dict_to_value(schemadict: &HashMap<&str, &Value>) -> Value {
+    let mut map = Map::new();
+    for (k, v) in schemadict {
+        map.insert(k.to_string(), (*v).clone());
+    }
+    Value::Object(map)
+}
+
+fn compile_contents_map(ctx: &Context, mut schemadict: HashMap<&str, &Value>) -> Result<Schema> {
+    ctx.increment()?;
+
     // We don't need to compile the schema if it's just meta and annotations
-    if schemadict
-        .keys()
-        .all(|k| META_AND_ANNOTATIONS.contains(&k.as_str()))
-    {
+    if schemadict.keys().all(|k| META_AND_ANNOTATIONS.contains(k)) {
         return Ok(Schema::Any);
     }
 
@@ -397,7 +412,7 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
 
     // Short-circuit for const -- don't need to compile the rest of the schema
     if let Some(instance) = schemadict.get("const") {
-        let validator = validator_for(contents)?;
+        let validator = validator_for(&dict_to_value(&schemadict))?;
         if let Some(instance) = instance_if_valid(instance, &validator) {
             return Ok(Schema::Const {
                 value: instance.clone(),
@@ -413,7 +428,7 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
 
     // Short-circuit for enum -- don't need to compile the rest of the schema
     if let Some(instances) = schemadict.get("enum") {
-        let validator: Validator = validator_for(contents)?;
+        let validator: Validator = validator_for(&dict_to_value(&schemadict))?;
         let instances = instances
             .as_array()
             .ok_or_else(|| anyhow!("enum must be an array"))?;
@@ -435,14 +450,11 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
         });
     }
 
-    // Make a mutable copy of the schema so we can modify it
-    let mut schemadict = schemadict.clone();
-
     if let Some(all_of) = schemadict.remove("allOf") {
         let all_of = all_of
             .as_array()
             .ok_or_else(|| anyhow!("allOf must be an array"))?;
-        let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
+        let siblings = compile_contents_map(ctx, schemadict)?;
         // Short-circuit if schema is already unsatisfiable
         if matches!(siblings, Schema::Unsatisfiable { .. }) {
             return Ok(siblings);
@@ -459,7 +471,7 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
         let any_of = any_of
             .as_array()
             .ok_or_else(|| anyhow!("anyOf must be an array"))?;
-        let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
+        let siblings = compile_contents_map(ctx, schemadict)?;
         // Short-circuit if schema is already unsatisfiable
         if matches!(siblings, Schema::Unsatisfiable { .. }) {
             return Ok(siblings);
@@ -477,7 +489,7 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
         let one_of = one_of
             .as_array()
             .ok_or_else(|| anyhow!("oneOf must be an array"))?;
-        let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
+        let siblings = compile_contents_map(ctx, schemadict)?;
         // Short-circuit if schema is already unsatisfiable
         if matches!(siblings, Schema::Unsatisfiable { .. }) {
             return Ok(siblings);
@@ -497,7 +509,7 @@ fn compile_contents_inner(ctx: &Context, contents: &Value) -> Result<Schema> {
             .to_string();
 
         let uri: String = ctx.normalize_ref(&reference)?.into_string();
-        let siblings = compile_contents(ctx, &Value::Object(schemadict))?;
+        let siblings = compile_contents_map(ctx, schemadict)?;
         if siblings == Schema::Any {
             define_ref(ctx, &uri)?;
             return Ok(Schema::Ref { uri });
@@ -556,37 +568,39 @@ fn intersect_ref(ctx: &Context, ref_uri: &str, schema: Schema) -> Result<Schema>
     intersect_two(ctx, schema, resolved_schema)
 }
 
-fn compile_type(ctx: &Context, tp: &str, schema: &Map<String, Value>) -> Result<Schema> {
+fn compile_type(ctx: &Context, tp: &str, schema: &HashMap<&str, &Value>) -> Result<Schema> {
     ctx.increment()?;
 
+    let get = |key: &str| schema.get(key).map(|v| *v);
+
     match tp {
         "null" => Ok(Schema::Null),
         "boolean" => Ok(Schema::Boolean),
         "number" | "integer" => compile_numeric(
-            schema.get("minimum"),
-            schema.get("maximum"),
-            schema.get("exclusiveMinimum"),
-            schema.get("exclusiveMaximum"),
+            get("minimum"),
+            get("maximum"),
+            get("exclusiveMinimum"),
+            get("exclusiveMaximum"),
             tp == "integer",
         ),
         "string" => compile_string(
-            schema.get("minLength"),
-            schema.get("maxLength"),
-            schema.get("pattern"),
-            schema.get("format"),
+            get("minLength"),
+            get("maxLength"),
+            get("pattern"),
+            get("format"),
         ),
         "array" => compile_array(
             ctx,
-            schema.get("minItems"),
-            schema.get("maxItems"),
-            schema.get("prefixItems"),
-            schema.get("items"),
+            get("minItems"),
+            get("maxItems"),
+            get("prefixItems"),
+            get("items"),
         ),
         "object" => compile_object(
             ctx,
-            schema.get("properties"),
-            schema.get("additionalProperties"),
-            schema.get("required"),
+            get("properties"),
+            get("additionalProperties"),
+            get("required"),
         ),
         _ => bail!("Invalid type: {}", tp),
     }

From 6cbfa13f08da500e1c0dadd7fceaaf561285c043 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 19 Nov 2024 15:37:29 -0800
Subject: [PATCH 70/73] anyhow-ify numeric regex builders

---
 parser/src/json/compiler.rs |  17 +-
 parser/src/json/numeric.rs  | 318 +++++++++++++++++++++---------------
 2 files changed, 200 insertions(+), 135 deletions(-)

diff --git a/parser/src/json/compiler.rs b/parser/src/json/compiler.rs
index e614941..12d41c9 100644
--- a/parser/src/json/compiler.rs
+++ b/parser/src/json/compiler.rs
@@ -1,4 +1,4 @@
-use anyhow::{anyhow, bail, Result};
+use anyhow::{anyhow, bail, Context, Result};
 use indexmap::IndexMap;
 use serde_json::{json, Value};
 use std::{collections::HashMap, vec};
@@ -339,7 +339,12 @@ impl Compiler {
         }
         .map(|val| val as i64);
         // TODO: handle errors in rx_int_range; currently it just panics
-        let rx = rx_int_range(minimum, maximum);
+        let rx = rx_int_range(minimum, maximum).with_context(|| {
+            format!(
+                "Failed to generate regex for integer range: min={:?}, max={:?}",
+                minimum, maximum
+            )
+        })?;
         Ok(self.lexeme(&rx))
     }
 
@@ -352,7 +357,13 @@ impl Compiler {
     ) -> Result<NodeRef> {
         check_number_bounds(minimum, maximum, exclusive_minimum, exclusive_maximum)?;
         // TODO: handle errors in rx_float_range; currently it just panics
-        let rx = rx_float_range(minimum, maximum, !exclusive_minimum, !exclusive_maximum);
+        let rx = rx_float_range(minimum, maximum, !exclusive_minimum, !exclusive_maximum)
+            .with_context(|| {
+                format!(
+                    "Failed to generate regex for float range: min={:?}, max={:?}",
+                    minimum, maximum
+                )
+            })?;
         Ok(self.lexeme(&rx))
     }
 
diff --git a/parser/src/json/numeric.rs b/parser/src/json/numeric.rs
index 79e8d8a..c6a090b 100644
--- a/parser/src/json/numeric.rs
+++ b/parser/src/json/numeric.rs
@@ -1,3 +1,5 @@
+use anyhow::{anyhow, Result};
+
 fn mk_or(parts: Vec<String>) -> String {
     if parts.len() == 1 {
         parts[0].clone()
@@ -10,53 +12,60 @@ fn num_digits(n: i64) -> usize {
     n.abs().to_string().len()
 }
 
-pub fn rx_int_range(left: Option<i64>, right: Option<i64>) -> String {
+pub fn rx_int_range(left: Option<i64>, right: Option<i64>) -> Result<String> {
     match (left, right) {
-        (None, None) => "-?(0|[1-9][0-9]*)".to_string(),
+        (None, None) => Ok("-?(0|[1-9][0-9]*)".to_string()),
         (Some(left), None) => {
             if left < 0 {
-                mk_or(vec![
-                    rx_int_range(Some(left), Some(-1)),
-                    rx_int_range(Some(0), None),
-                ])
+                Ok(mk_or(vec![
+                    rx_int_range(Some(left), Some(-1))?,
+                    rx_int_range(Some(0), None)?,
+                ]))
             } else {
-                mk_or(vec![
-                    rx_int_range(
-                        Some(left),
-                        Some("9".repeat(num_digits(left)).parse::<i64>().unwrap()),
-                    ),
+                let max_value = "9"
+                    .repeat(num_digits(left))
+                    .parse::<i64>()
+                    .map_err(|e| anyhow!("Failed to parse max value for left {}: {}", left, e))?;
+                Ok(mk_or(vec![
+                    rx_int_range(Some(left), Some(max_value))?,
                     format!("[1-9][0-9]{{{},}}", num_digits(left)),
-                ])
+                ]))
             }
         }
         (None, Some(right)) => {
             if right >= 0 {
-                mk_or(vec![
-                    rx_int_range(Some(0), Some(right)),
-                    rx_int_range(None, Some(-1)),
-                ])
+                Ok(mk_or(vec![
+                    rx_int_range(Some(0), Some(right))?,
+                    rx_int_range(None, Some(-1))?,
+                ]))
             } else {
-                format!("-{}", rx_int_range(Some(-right), None))
+                Ok(format!("-{}", rx_int_range(Some(-right), None)?))
             }
         }
         (Some(left), Some(right)) => {
-            assert!(left <= right);
+            if left > right {
+                return Err(anyhow!(
+                    "Invalid range: left ({}) cannot be greater than right ({})",
+                    left,
+                    right
+                ));
+            }
             if left < 0 {
                 if right < 0 {
-                    format!("(-{})", rx_int_range(Some(-right), Some(-left)))
+                    Ok(format!("(-{})", rx_int_range(Some(-right), Some(-left))?))
                 } else {
-                    format!(
+                    Ok(format!(
                         "(-{}|{})",
-                        rx_int_range(Some(0), Some(-left)),
-                        rx_int_range(Some(0), Some(right))
-                    )
+                        rx_int_range(Some(0), Some(-left))?,
+                        rx_int_range(Some(0), Some(right))?
+                    ))
                 }
             } else {
                 if num_digits(left) == num_digits(right) {
                     let l = left.to_string();
                     let r = right.to_string();
                     if left == right {
-                        return format!("({})", l);
+                        return Ok(format!("({})", l));
                     }
 
                     let lpref = &l[..l.len() - 1];
@@ -65,20 +74,19 @@ pub fn rx_int_range(left: Option<i64>, right: Option<i64>) -> String {
                     let rx = &r[r.len() - 1..];
 
                     if lpref == rpref {
-                        return format!("({}{}[{}-{}])", lpref, "", lx, rx);
+                        return Ok(format!("({}{}[{}-{}])", lpref, "", lx, rx));
+                    }
+
+                    let left_rec = lpref.parse::<i64>().unwrap_or(0);
+                    let right_rec = rpref.parse::<i64>().unwrap_or(0);
+                    if left_rec >= right_rec {
+                        return Err(anyhow!(
+                            "Invalid recursive range: left_rec ({}) must be less than right_rec ({})",
+                            left_rec,
+                            right_rec
+                        ));
                     }
 
-                    let left_rec: i64 = if lpref.is_empty() {
-                        0
-                    } else {
-                        lpref.parse().unwrap()
-                    };
-                    let right_rec: i64 = if rpref.is_empty() {
-                        0
-                    } else {
-                        rpref.parse().unwrap()
-                    };
-                    assert!(left_rec < right_rec);
                     let mut parts = Vec::new();
 
                     if lx != "0" {
@@ -90,109 +98,144 @@ pub fn rx_int_range(left: Option<i64>, right: Option<i64>) -> String {
                     }
 
                     if left_rec <= right_rec {
-                        let inner = rx_int_range(Some(left_rec), Some(right_rec));
+                        let inner = rx_int_range(Some(left_rec), Some(right_rec))?;
                         parts.push(format!("{}[0-9]", inner));
                     }
 
-                    mk_or(parts)
+                    Ok(mk_or(parts))
                 } else {
-                    let break_point = 10_i64.pow(num_digits(left) as u32) - 1;
-                    mk_or(vec![
-                        rx_int_range(Some(left), Some(break_point)),
-                        rx_int_range(Some(break_point + 1), Some(right)),
-                    ])
+                    let break_point = 10_i64
+                        .checked_pow(num_digits(left) as u32)
+                        .ok_or_else(|| anyhow!("Overflow when calculating break point"))?
+                        - 1;
+                    Ok(mk_or(vec![
+                        rx_int_range(Some(left), Some(break_point))?,
+                        rx_int_range(Some(break_point + 1), Some(right))?,
+                    ]))
                 }
             }
         }
     }
 }
 
-fn lexi_x_to_9(x: &str, incl: bool) -> String {
+fn lexi_x_to_9(x: &str, incl: bool) -> Result<String> {
     if incl {
         if x.is_empty() {
-            "[0-9]*".to_string()
+            Ok("[0-9]*".to_string())
         } else if x.len() == 1 {
-            format!("[{}-9][0-9]*", x)
+            Ok(format!("[{}-9][0-9]*", x))
         } else {
-            let x0 = x.chars().next().unwrap().to_digit(10).unwrap();
+            let x0 = x.chars()
+                .next()
+                .ok_or_else(|| anyhow!("String x is unexpectedly empty"))?
+                .to_digit(10)
+                .ok_or_else(|| anyhow!("Failed to parse character as digit"))?;
             let x_rest = &x[1..];
             let mut parts = vec![format!(
                 "{}{}",
-                x.chars().next().unwrap(),
-                lexi_x_to_9(x_rest, incl)
+                x.chars().next().ok_or_else(|| anyhow!("String x is unexpectedly empty"))?,
+                lexi_x_to_9(x_rest, incl)?
             )];
             if x0 < 9 {
                 parts.push(format!("[{}-9][0-9]*", x0 + 1));
             }
-            mk_or(parts)
+            Ok(mk_or(parts))
         }
     } else {
         if x.is_empty() {
-            "[0-9]*[1-9]".to_string()
+            Ok("[0-9]*[1-9]".to_string())
         } else {
-            let x0 = x.chars().next().unwrap().to_digit(10).unwrap();
+            let x0 = x.chars()
+                .next()
+                .ok_or_else(|| anyhow!("String x is unexpectedly empty"))?
+                .to_digit(10)
+                .ok_or_else(|| anyhow!("Failed to parse character as digit"))?;
             let x_rest = &x[1..];
             let mut parts = vec![format!(
                 "{}{}",
-                x.chars().next().unwrap(),
-                lexi_x_to_9(x_rest, incl)
+                x.chars().next().ok_or_else(|| anyhow!("String x is unexpectedly empty"))?,
+                lexi_x_to_9(x_rest, incl)?
             )];
             if x0 < 9 {
                 parts.push(format!("[{}-9][0-9]*", x0 + 1));
             }
-            mk_or(parts)
+            Ok(mk_or(parts))
         }
     }
 }
 
-fn lexi_0_to_x(x: &str, incl: bool) -> String {
+fn lexi_0_to_x(x: &str, incl: bool) -> Result<String> {
     if x.is_empty() {
-        assert!(incl);
-        "".to_string()
+        if incl {
+            Ok("".to_string())
+        } else {
+            Err(anyhow!("Inclusive flag must be true for an empty string"))
+        }
     } else {
-        let x0 = x.chars().next().unwrap().to_digit(10).unwrap();
+        let x0 = x.chars()
+            .next()
+            .ok_or_else(|| anyhow!("String x is unexpectedly empty"))?
+            .to_digit(10)
+            .ok_or_else(|| anyhow!("Failed to parse character as digit"))?;
         let x_rest = &x[1..];
 
         if !incl && x.len() == 1 {
-            assert!(x0 > 0);
-            return format!("[0-{}][0-9]*", x0 - 1);
+            if x0 == 0 {
+                return Err(anyhow!("x0 must be greater than 0 for non-inclusive single character"));
+            }
+            return Ok(format!("[0-{}][0-9]*", x0 - 1));
         }
 
         let mut parts = vec![format!(
             "{}{}",
-            x.chars().next().unwrap(),
-            lexi_0_to_x(x_rest, incl)
+            x.chars().next().ok_or_else(|| anyhow!("String x is unexpectedly empty"))?,
+            lexi_0_to_x(x_rest, incl)?
         )];
         if x0 > 0 {
             parts.push(format!("[0-{}][0-9]*", x0 - 1));
         }
-        mk_or(parts)
+        Ok(mk_or(parts))
     }
 }
 
-fn lexi_range(ld: &str, rd: &str, ld_incl: bool, rd_incl: bool) -> String {
-    assert_eq!(ld.len(), rd.len());
+fn lexi_range(ld: &str, rd: &str, ld_incl: bool, rd_incl: bool) -> Result<String> {
+    if ld.len() != rd.len() {
+        return Err(anyhow!("ld and rd must have the same length"));
+    }
     if ld == rd {
-        assert!(ld_incl && rd_incl);
-        ld.to_string()
+        if ld_incl && rd_incl {
+            Ok(ld.to_string())
+        } else {
+            Err(anyhow!("Empty range when ld equals rd and not both inclusive"))
+        }
     } else {
-        let l0 = ld.chars().next().unwrap().to_digit(10).unwrap();
-        let r0 = rd.chars().next().unwrap().to_digit(10).unwrap();
+        let l0 = ld.chars()
+            .next()
+            .ok_or_else(|| anyhow!("ld is unexpectedly empty"))?
+            .to_digit(10)
+            .ok_or_else(|| anyhow!("Failed to parse character as digit"))?;
+        let r0 = rd.chars()
+            .next()
+            .ok_or_else(|| anyhow!("rd is unexpectedly empty"))?
+            .to_digit(10)
+            .ok_or_else(|| anyhow!("Failed to parse character as digit"))?;
         if l0 == r0 {
             let ld_rest = &ld[1..];
             let rd_rest = &rd[1..];
-            format!(
+            Ok(format!(
                 "{}{}",
-                ld.chars().next().unwrap(),
-                lexi_range(ld_rest, rd_rest, ld_incl, rd_incl)
-            )
+                ld.chars().next().ok_or_else(|| anyhow!("ld is unexpectedly empty"))?,
+                lexi_range(ld_rest, rd_rest, ld_incl, rd_incl)?
+            ))
         } else {
-            assert!(l0 < r0);
+            if l0 >= r0 {
+                return Err(anyhow!("l0 must be less than r0"));
+            }
             let ld_rest = ld[1..].trim_end_matches('0');
             let mut parts = vec![format!(
                 "{}{}",
-                ld.chars().next().unwrap(),
-                lexi_x_to_9(ld_rest, ld_incl)
+                ld.chars().next().ok_or_else(|| anyhow!("ld is unexpectedly empty"))?,
+                lexi_x_to_9(ld_rest, ld_incl)?
             )];
             if l0 + 1 < r0 {
                 parts.push(format!("[{}-{}][0-9]*", l0 + 1, r0 - 1));
@@ -201,11 +244,11 @@ fn lexi_range(ld: &str, rd: &str, ld_incl: bool, rd_incl: bool) -> String {
             if !rd_rest.is_empty() || rd_incl {
                 parts.push(format!(
                     "{}{}",
-                    rd.chars().next().unwrap(),
-                    lexi_0_to_x(rd_rest, rd_incl)
+                    rd.chars().next().ok_or_else(|| anyhow!("rd is unexpectedly empty"))?,
+                    lexi_0_to_x(rd_rest, rd_incl)?
                 ));
             }
-            mk_or(parts)
+            Ok(mk_or(parts))
         }
     }
 }
@@ -219,95 +262,106 @@ pub fn rx_float_range(
     right: Option<f64>,
     left_inclusive: bool,
     right_inclusive: bool,
-) -> String {
+) -> Result<String> {
     match (left, right) {
-        (None, None) => "-?(0|[1-9][0-9]*)(\\.[0-9]+)?([eE][+-]?[0-9]+)?".to_string(),
+        (None, None) => Ok("-?(0|[1-9][0-9]*)(\\.[0-9]+)?([eE][+-]?[0-9]+)?".to_string()),
         (Some(left), None) => {
             if left < 0.0 {
-                mk_or(vec![
-                    rx_float_range(Some(left), Some(0.0), left_inclusive, false),
-                    rx_float_range(Some(0.0), None, true, false),
-                ])
+                Ok(mk_or(vec![
+                    rx_float_range(Some(left), Some(0.0), left_inclusive, false)?,
+                    rx_float_range(Some(0.0), None, true, false)?,
+                ]))
             } else {
                 let left_int_part = left as i64;
-                mk_or(vec![
+                Ok(mk_or(vec![
                     rx_float_range(
                         Some(left),
                         Some(10f64.powi(num_digits(left_int_part) as i32)),
                         left_inclusive,
                         false,
-                    ),
+                    )?,
                     format!("[1-9][0-9]{{{},}}(\\.[0-9]+)?", num_digits(left_int_part)),
-                ])
+                ]))
             }
         }
         (None, Some(right)) => {
             if right == 0.0 {
-                let r = format!("-{}", rx_float_range(Some(0.0), None, false, false));
+                let r = format!("-{}", rx_float_range(Some(0.0), None, false, false)?);
                 if right_inclusive {
-                    mk_or(vec![r, "0".to_string()])
+                    Ok(mk_or(vec![r, "0".to_string()]))
                 } else {
-                    r
+                    Ok(r)
                 }
             } else if right > 0.0 {
-                mk_or(vec![
-                    format!("-{}", rx_float_range(Some(0.0), None, false, false)),
-                    rx_float_range(Some(0.0), Some(right), true, right_inclusive),
-                ])
+                Ok(mk_or(vec![
+                    format!("-{}", rx_float_range(Some(0.0), None, false, false)?),
+                    rx_float_range(Some(0.0), Some(right), true, right_inclusive)?,
+                ]))
             } else {
-                format!(
+                Ok(format!(
                     "-{}",
-                    rx_float_range(Some(-right), None, right_inclusive, false)
-                )
+                    rx_float_range(Some(-right), None, right_inclusive, false)?
+                ))
             }
         }
         (Some(left), Some(right)) => {
-            assert!(left <= right);
+            if left > right {
+                return Err(anyhow!(
+                    "Invalid range: left ({}) cannot be greater than right ({})",
+                    left,
+                    right
+                ));
+            }
             if left == right {
                 if left_inclusive && right_inclusive {
-                    format!("({})", float_to_str(left))
+                    Ok(format!("({})", float_to_str(left)))
                 } else {
-                    panic!("Empty range");
+                    Err(anyhow!(
+                        "Empty range when left equals right and not both inclusive"
+                    ))
                 }
             } else if left < 0.0 {
                 if right < 0.0 {
-                    format!(
+                    Ok(format!(
                         "(-{})",
-                        rx_float_range(Some(-right), Some(-left), right_inclusive, left_inclusive)
-                    )
+                        rx_float_range(Some(-right), Some(-left), right_inclusive, left_inclusive)?
+                    ))
                 } else {
                     let mut parts = vec![];
-                    let neg_part = rx_float_range(
-                        Some(0.0),
-                        Some(-left),
-                        false, // we don't allow -0
-                        left_inclusive,
-                    );
+                    let neg_part = rx_float_range(Some(0.0), Some(-left), false, left_inclusive)?;
                     parts.push(format!("(-{})", neg_part));
 
                     if right > 0.0 || right_inclusive {
-                        let pos_part = rx_float_range(
-                            Some(0.0),
-                            Some(right),
-                            true, // always include 0
-                            right_inclusive,
-                        );
+                        let pos_part =
+                            rx_float_range(Some(0.0), Some(right), true, right_inclusive)?;
                         parts.push(pos_part);
                     }
-                    mk_or(parts)
+                    Ok(mk_or(parts))
                 }
             } else {
                 let l = float_to_str(left);
                 let r = float_to_str(right);
-                assert!(l != r);
-                assert!(!l.contains('e') && !r.contains('e'));
-
+                if l == r {
+                    return Err(anyhow!(
+                        "Unexpected equality of left and right string representations"
+                    ));
+                }
                 if !left.is_finite() || !right.is_finite() {
-                    panic!("Infinite numbers not supported");
+                    return Err(anyhow!("Infinite numbers not supported"));
                 }
 
-                let mut left_rec: i64 = l.split('.').next().unwrap().parse().unwrap();
-                let right_rec: i64 = r.split('.').next().unwrap().parse().unwrap();
+                let mut left_rec: i64 = l
+                    .split('.')
+                    .next()
+                    .ok_or_else(|| anyhow!("Failed to split left integer part"))?
+                    .parse()
+                    .map_err(|e| anyhow!("Failed to parse left integer part: {}", e))?;
+                let right_rec: i64 = r
+                    .split('.')
+                    .next()
+                    .ok_or_else(|| anyhow!("Failed to split right integer part"))?
+                    .parse()
+                    .map_err(|e| anyhow!("Failed to parse right integer part: {}", e))?;
 
                 let mut ld = l.split('.').nth(1).unwrap_or("").to_string();
                 let mut rd = r.split('.').nth(1).unwrap_or("").to_string();
@@ -321,12 +375,12 @@ pub fn rx_float_range(
                     }
                     let suff = format!(
                         "\\.{}",
-                        lexi_range(&ld, &rd, left_inclusive, right_inclusive)
+                        lexi_range(&ld, &rd, left_inclusive, right_inclusive)?
                     );
-                    if ld.parse::<i64>().unwrap() == 0 {
-                        format!("({}({})?)", left_rec, suff)
+                    if ld.parse::<i64>().unwrap_or(0) == 0 {
+                        Ok(format!("({}({})?)", left_rec, suff))
                     } else {
-                        format!("({}{})", left_rec, suff)
+                        Ok(format!("({}{})", left_rec, suff))
                     }
                 } else {
                     let mut parts = vec![];
@@ -334,13 +388,13 @@ pub fn rx_float_range(
                         parts.push(format!(
                             "({}\\.{})",
                             left_rec,
-                            lexi_x_to_9(&ld, left_inclusive)
+                            lexi_x_to_9(&ld, left_inclusive)?
                         ));
                         left_rec += 1;
                     }
 
                     if right_rec - 1 >= left_rec {
-                        let inner = rx_int_range(Some(left_rec), Some(right_rec - 1));
+                        let inner = rx_int_range(Some(left_rec), Some(right_rec - 1))?;
                         parts.push(format!("({}(\\.[0-9]+)?)", inner));
                     }
 
@@ -348,13 +402,13 @@ pub fn rx_float_range(
                         parts.push(format!(
                             "({}(\\.{})?)",
                             right_rec,
-                            lexi_0_to_x(&rd, right_inclusive)
+                            lexi_0_to_x(&rd, right_inclusive)?
                         ));
                     } else if right_inclusive {
                         parts.push(format!("{}(\\.0+)?", right_rec));
                     }
 
-                    mk_or(parts)
+                    Ok(mk_or(parts))
                 }
             }
         }

From 20004c3a438df32ff82a5ecaed3a5a4ccc26006b Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 19 Nov 2024 16:32:34 -0800
Subject: [PATCH 71/73] cargo fmt

---
 parser/src/json/numeric.rs | 47 +++++++++++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 13 deletions(-)

diff --git a/parser/src/json/numeric.rs b/parser/src/json/numeric.rs
index c6a090b..8c33f55 100644
--- a/parser/src/json/numeric.rs
+++ b/parser/src/json/numeric.rs
@@ -125,7 +125,8 @@ fn lexi_x_to_9(x: &str, incl: bool) -> Result<String> {
         } else if x.len() == 1 {
             Ok(format!("[{}-9][0-9]*", x))
         } else {
-            let x0 = x.chars()
+            let x0 = x
+                .chars()
                 .next()
                 .ok_or_else(|| anyhow!("String x is unexpectedly empty"))?
                 .to_digit(10)
@@ -133,7 +134,9 @@ fn lexi_x_to_9(x: &str, incl: bool) -> Result<String> {
             let x_rest = &x[1..];
             let mut parts = vec![format!(
                 "{}{}",
-                x.chars().next().ok_or_else(|| anyhow!("String x is unexpectedly empty"))?,
+                x.chars()
+                    .next()
+                    .ok_or_else(|| anyhow!("String x is unexpectedly empty"))?,
                 lexi_x_to_9(x_rest, incl)?
             )];
             if x0 < 9 {
@@ -145,7 +148,8 @@ fn lexi_x_to_9(x: &str, incl: bool) -> Result<String> {
         if x.is_empty() {
             Ok("[0-9]*[1-9]".to_string())
         } else {
-            let x0 = x.chars()
+            let x0 = x
+                .chars()
                 .next()
                 .ok_or_else(|| anyhow!("String x is unexpectedly empty"))?
                 .to_digit(10)
@@ -153,7 +157,9 @@ fn lexi_x_to_9(x: &str, incl: bool) -> Result<String> {
             let x_rest = &x[1..];
             let mut parts = vec![format!(
                 "{}{}",
-                x.chars().next().ok_or_else(|| anyhow!("String x is unexpectedly empty"))?,
+                x.chars()
+                    .next()
+                    .ok_or_else(|| anyhow!("String x is unexpectedly empty"))?,
                 lexi_x_to_9(x_rest, incl)?
             )];
             if x0 < 9 {
@@ -172,7 +178,8 @@ fn lexi_0_to_x(x: &str, incl: bool) -> Result<String> {
             Err(anyhow!("Inclusive flag must be true for an empty string"))
         }
     } else {
-        let x0 = x.chars()
+        let x0 = x
+            .chars()
             .next()
             .ok_or_else(|| anyhow!("String x is unexpectedly empty"))?
             .to_digit(10)
@@ -181,14 +188,18 @@ fn lexi_0_to_x(x: &str, incl: bool) -> Result<String> {
 
         if !incl && x.len() == 1 {
             if x0 == 0 {
-                return Err(anyhow!("x0 must be greater than 0 for non-inclusive single character"));
+                return Err(anyhow!(
+                    "x0 must be greater than 0 for non-inclusive single character"
+                ));
             }
             return Ok(format!("[0-{}][0-9]*", x0 - 1));
         }
 
         let mut parts = vec![format!(
             "{}{}",
-            x.chars().next().ok_or_else(|| anyhow!("String x is unexpectedly empty"))?,
+            x.chars()
+                .next()
+                .ok_or_else(|| anyhow!("String x is unexpectedly empty"))?,
             lexi_0_to_x(x_rest, incl)?
         )];
         if x0 > 0 {
@@ -206,15 +217,19 @@ fn lexi_range(ld: &str, rd: &str, ld_incl: bool, rd_incl: bool) -> Result<String
         if ld_incl && rd_incl {
             Ok(ld.to_string())
         } else {
-            Err(anyhow!("Empty range when ld equals rd and not both inclusive"))
+            Err(anyhow!(
+                "Empty range when ld equals rd and not both inclusive"
+            ))
         }
     } else {
-        let l0 = ld.chars()
+        let l0 = ld
+            .chars()
             .next()
             .ok_or_else(|| anyhow!("ld is unexpectedly empty"))?
             .to_digit(10)
             .ok_or_else(|| anyhow!("Failed to parse character as digit"))?;
-        let r0 = rd.chars()
+        let r0 = rd
+            .chars()
             .next()
             .ok_or_else(|| anyhow!("rd is unexpectedly empty"))?
             .to_digit(10)
@@ -224,7 +239,9 @@ fn lexi_range(ld: &str, rd: &str, ld_incl: bool, rd_incl: bool) -> Result<String
             let rd_rest = &rd[1..];
             Ok(format!(
                 "{}{}",
-                ld.chars().next().ok_or_else(|| anyhow!("ld is unexpectedly empty"))?,
+                ld.chars()
+                    .next()
+                    .ok_or_else(|| anyhow!("ld is unexpectedly empty"))?,
                 lexi_range(ld_rest, rd_rest, ld_incl, rd_incl)?
             ))
         } else {
@@ -234,7 +251,9 @@ fn lexi_range(ld: &str, rd: &str, ld_incl: bool, rd_incl: bool) -> Result<String
             let ld_rest = ld[1..].trim_end_matches('0');
             let mut parts = vec![format!(
                 "{}{}",
-                ld.chars().next().ok_or_else(|| anyhow!("ld is unexpectedly empty"))?,
+                ld.chars()
+                    .next()
+                    .ok_or_else(|| anyhow!("ld is unexpectedly empty"))?,
                 lexi_x_to_9(ld_rest, ld_incl)?
             )];
             if l0 + 1 < r0 {
@@ -244,7 +263,9 @@ fn lexi_range(ld: &str, rd: &str, ld_incl: bool, rd_incl: bool) -> Result<String
             if !rd_rest.is_empty() || rd_incl {
                 parts.push(format!(
                     "{}{}",
-                    rd.chars().next().ok_or_else(|| anyhow!("rd is unexpectedly empty"))?,
+                    rd.chars()
+                        .next()
+                        .ok_or_else(|| anyhow!("rd is unexpectedly empty"))?,
                     lexi_0_to_x(rd_rest, rd_incl)?
                 ));
             }

From ba506428ce3f401af3ef5e693bc40b7fe83571f8 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 19 Nov 2024 16:33:43 -0800
Subject: [PATCH 72/73] depend on jsonschema_validation feature

---
 parser/src/json/schema.rs | 108 +++++++++++++++++++++++---------------
 1 file changed, 67 insertions(+), 41 deletions(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index 54fb5ec..2799930 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -6,7 +6,6 @@ use std::{
 
 use anyhow::{anyhow, bail, Result};
 use indexmap::{IndexMap, IndexSet};
-use jsonschema::{validator_for, Uri, Validator};
 use referencing::{Draft, Registry, Resolver, ResourceRef};
 use serde_json::{Map, Value};
 
@@ -260,11 +259,12 @@ impl<'a> Context<'a> {
             .create_resource_ref(contents)
     }
 
-    fn normalize_ref(&self, reference: &str) -> Result<Uri<String>> {
+    fn normalize_ref(&self, reference: &str) -> Result<String> {
         Ok(self
             .resolver
             .resolve_against(&self.resolver.base_uri().borrow(), reference)?
-            .normalize())
+            .normalize()
+            .into_string())
     }
 
     fn lookup_resource(&'a self, reference: &str) -> Result<ResourceRef> {
@@ -315,14 +315,6 @@ fn draft_for(value: &Value) -> Draft {
     DEFAULT_DRAFT.detect(value).unwrap_or(DEFAULT_DRAFT)
 }
 
-fn instance_if_valid<'a>(instance: &'a Value, validator: &'a Validator) -> Option<&'a Value> {
-    if validator.is_valid(instance) {
-        Some(instance)
-    } else {
-        None
-    }
-}
-
 pub fn build_schema(contents: Value) -> Result<(Schema, HashMap<String, Schema>)> {
     if let Some(b) = contents.as_bool() {
         if b {
@@ -393,11 +385,15 @@ fn dict_to_value(schemadict: &HashMap<&str, &Value>) -> Value {
     Value::Object(map)
 }
 
+fn only_meta_and_annotations(schemadict: &HashMap<&str, &Value>) -> bool {
+    schemadict.keys().all(|k| META_AND_ANNOTATIONS.contains(k))
+}
+
 fn compile_contents_map(ctx: &Context, mut schemadict: HashMap<&str, &Value>) -> Result<Schema> {
     ctx.increment()?;
 
     // We don't need to compile the schema if it's just meta and annotations
-    if schemadict.keys().all(|k| META_AND_ANNOTATIONS.contains(k)) {
+    if only_meta_and_annotations(&schemadict) {
         return Ok(Schema::Any);
     }
 
@@ -411,43 +407,73 @@ fn compile_contents_map(ctx: &Context, mut schemadict: HashMap<&str, &Value>) ->
     }
 
     // Short-circuit for const -- don't need to compile the rest of the schema
-    if let Some(instance) = schemadict.get("const") {
-        let validator = validator_for(&dict_to_value(&schemadict))?;
-        if let Some(instance) = instance_if_valid(instance, &validator) {
+    if let Some(instance) = schemadict.remove("const") {
+        if only_meta_and_annotations(&schemadict) {
             return Ok(Schema::Const {
                 value: instance.clone(),
             });
         }
-        return Ok(Schema::Unsatisfiable {
-            reason: format!(
-                "const instance is invalid against parent schema: {:?}",
-                instance
-            ),
-        });
-    }
-
-    // Short-circuit for enum -- don't need to compile the rest of the schema
-    if let Some(instances) = schemadict.get("enum") {
-        let validator: Validator = validator_for(&dict_to_value(&schemadict))?;
-        let instances = instances
-            .as_array()
-            .ok_or_else(|| anyhow!("enum must be an array"))?;
-        let valid_instances = instances
-            .iter()
-            .filter_map(|instance| instance_if_valid(instance, &validator))
-            .cloned()
-            .collect::<Vec<_>>();
-        if valid_instances.is_empty() {
+        #[cfg(not(feature = "jsonschema_validation"))]
+        {
+            return Err(anyhow!(
+                "const keyword with siblings requires jsonschema_validation feature"
+            ));
+        }
+        #[cfg(feature = "jsonschema_validation")]
+        {
+            use jsonschema::validator_for;
+            let validator = validator_for(&dict_to_value(&schemadict))?;
+            if validator.is_valid(instance) {
+                return Ok(Schema::Const {
+                    value: instance.clone(),
+                });
+            }
             return Ok(Schema::Unsatisfiable {
                 reason: format!(
-                    "enum instances all invalid against parent schema: {:?}",
-                    instances
+                    "const instance is invalid against parent schema: {:?}",
+                    instance
                 ),
             });
         }
-        return Ok(Schema::Enum {
-            options: valid_instances,
-        });
+    }
+
+    // Short-circuit for enum -- don't need to compile the rest of the schema
+    if let Some(instances) = schemadict.remove("enum") {
+        if only_meta_and_annotations(&schemadict) {
+            return Ok(Schema::Enum {
+                options: instances
+                    .as_array()
+                    .ok_or_else(|| anyhow!("enum must be an array"))?
+                    .clone(),
+            });
+        }
+        #[cfg(not(feature = "jsonschema_validation"))]
+        {
+            return Err(anyhow!(
+                "enum keyword with siblings requires jsonschema_validation feature"
+            ));
+        }
+        #[cfg(feature = "jsonschema_validation")]
+        {
+            use jsonschema::validator_for;
+            let validator = validator_for(&dict_to_value(&schemadict))?;
+            let (valid, invalid): (Vec<_>, Vec<_>) = instances
+                .as_array()
+                .ok_or_else(|| anyhow!("enum must be an array"))?
+                .into_iter()
+                .partition(|instance| validator.is_valid(instance));
+            if valid.is_empty() {
+                return Ok(Schema::Unsatisfiable {
+                    reason: format!(
+                        "enum instances all invalid against parent schema: {:?}",
+                        invalid
+                    ),
+                });
+            }
+            return Ok(Schema::Enum {
+                options: valid.into_iter().cloned().collect(),
+            });
+        }
     }
 
     if let Some(all_of) = schemadict.remove("allOf") {
@@ -508,7 +534,7 @@ fn compile_contents_map(ctx: &Context, mut schemadict: HashMap<&str, &Value>) ->
             .ok_or_else(|| anyhow!("$ref must be a string, got {}", limited_str(&reference)))?
             .to_string();
 
-        let uri: String = ctx.normalize_ref(&reference)?.into_string();
+        let uri: String = ctx.normalize_ref(&reference)?;
         let siblings = compile_contents_map(ctx, schemadict)?;
         if siblings == Schema::Any {
             define_ref(ctx, &uri)?;

From adac6bceecfb8edcc8bef8bbf0e61220fd4c8936 Mon Sep 17 00:00:00 2001
From: Hudson Cooper <hncooper96@gmail.com>
Date: Tue, 19 Nov 2024 16:34:38 -0800
Subject: [PATCH 73/73] remove non-tests

---
 parser/src/json/schema.rs | 145 --------------------------------------
 1 file changed, 145 deletions(-)

diff --git a/parser/src/json/schema.rs b/parser/src/json/schema.rs
index 2799930..4e76798 100644
--- a/parser/src/json/schema.rs
+++ b/parser/src/json/schema.rs
@@ -1045,148 +1045,3 @@ fn opt_min<T: PartialOrd>(a: Option<T>, b: Option<T>) -> Option<T> {
         (None, None) => None,
     }
 }
-
-mod test {
-    use super::*;
-    use serde_json::json;
-
-    #[test]
-    fn test_linked_list_simple() {
-        let schema = json!({
-            "type": "object",
-            "properties": {
-                "value": {
-                    "type": "number",
-                },
-                "next": {
-                    "$ref": "#",
-                },
-            },
-            "additionalProperties": false,
-            "required": ["value"],
-        });
-        let schema = build_schema(&schema).unwrap();
-        println!("{:?}", schema);
-    }
-
-    #[test]
-    fn test_normalize_3() {
-        let schema = json!({
-            "anyOf": [
-                {
-                    "type": "string",
-                    "minLength": 1,
-                },
-                {
-                    "type": "number",
-                    "minimum": 10,
-                },
-            ],
-            "oneOf": [
-                {
-                    "type": ["string", "integer"],
-                    "maxLength": 1,
-                },
-                {
-                    "type": "number",
-                    "maximum": 11,
-                },
-            ],
-            "pattern": "^[a-z]+$",
-        });
-        let schema = build_schema(&schema).unwrap();
-        println!("{:?}", schema);
-    }
-
-    #[test]
-    fn test_ref() {
-        let schema = json!({
-            "type": "object",
-            "properties": {
-                "foo": {
-                    "$ref": "#ooo",
-                },
-                "bar": {
-                    "$id": "baz.com",
-                    "type": "array",
-                    "prefixItems": [
-                        {
-                            "$ref": "#/prefixItems/1"
-                        },
-                        {
-                            "type": "number",
-                        },
-                    ],
-                },
-                "baz": {
-                    "$anchor": "ooo",
-                    "type": "number",
-                },
-            },
-        });
-        let schema = build_schema(&schema).unwrap();
-        println!("{:?}", schema);
-    }
-
-    #[test]
-    fn test_fhir() -> Result<()> {
-        let file = std::fs::read_to_string("../../fhir.schema.json")?;
-        let schema = build_schema(&serde_json::from_str(&file)?)?;
-        println!("{:?}", schema.0);
-        Ok(())
-    }
-
-    #[test]
-    #[ignore]
-    fn test_recursive_ref() {
-        let schema = json!({
-            "type": "object",
-            "properties": {
-                "value": {
-                    "type": "number",
-                },
-                "next": {
-                    "$ref": "#",
-                    "properties": {
-                        "value": {
-                            "type": "integer",
-                        },
-                    }
-                },
-            },
-            "additionalProperties": false,
-            "required": ["value"],
-        });
-        let schema = build_schema(&schema).unwrap();
-        println!("{:?}", schema);
-    }
-
-    #[test]
-    fn test_indirect_recursive_ref() {
-        let schema = json!({
-            "type": "object",
-            "properties": {
-                "value": {
-                    "type": "number",
-                },
-                "next": {
-                    "$ref": "#/$defs/inner",
-                },
-            },
-            "additionalProperties": false,
-            "required": ["value"],
-            "$defs": {
-                "inner": {
-                    "$ref": "#",
-                    "properties": {
-                        "value": {
-                            "type": "integer",
-                        },
-                    }
-                }
-            }
-        });
-        let schema = build_schema(&schema).unwrap();
-        println!("{:?}", schema);
-    }
-}