feat!: resolve declaration names before the resolver (#4353)

PRQL · Mar 25, 2024 · 125aafb · 125aafb
1 parent 7dbe057
commit 125aafb
Show file tree

Hide file tree

Showing 66 changed files with 1,838 additions and 1,145 deletions.
diff --git a/.config/nextest.toml b/.config/nextest.toml
@@ -1,6 +1,7 @@
 [profile.default]
 fail-fast = false
 failure-output = "final"
+status-level = "slow"
 slow-timeout = {period = "500ms", terminate-after = 4}
 
 [[profile.default.overrides]]

diff --git a/prqlc/bindings/js/README.md b/prqlc/bindings/js/README.md
@@ -76,7 +76,7 @@ console.log(sql);
 <html>
   <head>
     <script type="module">
-      import init, { compile } from './dist/web/prql_js.js';
+      import init, { compile } from "./dist/web/prql_js.js";
       await init();
 
       const sql = compile("from db.employees | select first_name");

diff --git a/prqlc/bindings/js/tests/test_all.mjs b/prqlc/bindings/js/tests/test_all.mjs
@@ -94,7 +94,10 @@ describe("prql-js", () => {
       const opts = new prql.CompileOptions();
 
       opts.target = "sql.any";
-      const res = prql.compile("prql target:sql.mssql\nfrom db.a | take 1", opts);
+      const res = prql.compile(
+        "prql target:sql.mssql\nfrom db.a | take 1",
+        opts,
+      );
       assert(res.includes("1 ROWS ONLY"));
     });
   });

diff --git a/prqlc/prqlc-ast/src/error.rs b/prqlc/prqlc-ast/src/error.rs
@@ -146,6 +146,7 @@ pub trait WithErrorInfo: Sized {
     fn with_hints<S: Into<String>, I: IntoIterator<Item = S>>(self, hints: I) -> Self;
 
     fn with_span(self, span: Option<Span>) -> Self;
+    fn with_span_fallback(self, span: Option<Span>) -> Self;
     fn with_code(self, code: &'static str) -> Self;
 }
 
@@ -160,6 +161,11 @@ impl WithErrorInfo for Error {
         self
     }
 
+    fn with_span_fallback(mut self, span: Option<Span>) -> Self {
+        self.span = self.span.or(span);
+        self
+    }
+
     fn push_hint<S: Into<String>>(mut self, hint: S) -> Self {
         self.hints.push(hint.into());
         self
@@ -180,6 +186,10 @@ impl<T, E: WithErrorInfo> WithErrorInfo for Result<T, E> {
         self.map_err(|e| e.with_span(span))
     }
 
+    fn with_span_fallback(self, span: Option<Span>) -> Self {
+        self.map_err(|e| e.with_span_fallback(span))
+    }
+
     fn with_code(self, code: &'static str) -> Self {
         self.map_err(|e| e.with_code(code))
     }

diff --git a/prqlc/prqlc-ast/src/expr/ident.rs b/prqlc/prqlc-ast/src/expr/ident.rs
@@ -144,7 +144,11 @@ impl<'de> Deserialize<'de> for Ident {
 }
 
 pub fn display_ident(f: &mut std::fmt::Formatter, ident: &Ident) -> Result<(), std::fmt::Error> {
-    for part in &ident.path {
+    let mut path = &ident.path[..];
+    if path.first().map_or(false, |f| f == "_local") {
+        path = &path[1..];
+    }
+    for part in path {
         display_ident_part(f, part)?;
         f.write_char('.')?;
     }

diff --git a/prqlc/prqlc/src/cli/mod.rs b/prqlc/prqlc/src/cli/mod.rs
@@ -291,6 +291,7 @@ impl Command {
                 let mut root_module_def = prql_to_pl_tree(sources)?;
 
                 drop_module_def(&mut root_module_def.stmts, "std");
+                drop_module_def(&mut root_module_def.stmts, "_local");
 
                 pl_to_prql(&root_module_def)?.into_bytes()
             }
@@ -302,6 +303,7 @@ impl Command {
                 let mut restricted = prqlc::semantic::ast_expand::restrict_module_def(expanded);
 
                 drop_module_def(&mut restricted.stmts, "std");
+                drop_module_def(&mut restricted.stmts, "_local");
 
                 pl_to_prql(&restricted)?.into_bytes()
             }
@@ -323,6 +325,7 @@ impl Command {
                 // resolved PL, restricted back into AST
                 let mut root_module = semantic::ast_expand::restrict_module(root_module.module);
                 drop_module_def(&mut root_module.stmts, "std");
+                drop_module_def(&mut root_module.stmts, "_local");
                 out.extend(pl_to_prql(&root_module)?.into_bytes());
 
                 out
@@ -638,7 +641,10 @@ sort full
             },
             &mut SourceTree::new(
                 [
-                    ("Project.prql".into(), "orders.x | select y".to_string()),
+                    (
+                        "Project.prql".into(),
+                        "project.orders.x | select y".to_string(),
+                    ),
                     (
                         "orders.prql".into(),
                         "let x = (from db.z | select {y, u})".to_string(),

diff --git a/prqlc/prqlc/src/codegen/ast.rs b/prqlc/prqlc/src/codegen/ast.rs
@@ -292,7 +292,12 @@ impl WriteSource for Ident {
         opt.consume_width(width as u16)?;
 
         let mut r = String::new();
-        for part in &self.path {
+
+        let mut path = &self.path[..];
+        if path.first().map_or(false, |f| f == "_local") {
+            path = &path[1..];
+        }
+        for part in path {
             r += &write_ident_part(part);
             r += ".";
         }
@@ -303,7 +308,9 @@ impl WriteSource for Ident {
 
 pub static KEYWORDS: Lazy<HashSet<&str>> = Lazy::new(|| {
     HashSet::from_iter([
-        "let", "into", "case", "prql", "type", "module", "internal", "func",
+        "let", "into", "case", "prql", "type", "internal",
+        "func",
+        // "module" can be both keyword and ident
     ])
 });
 

diff --git a/prqlc/prqlc/src/ir/decl.rs b/prqlc/prqlc/src/ir/decl.rs
@@ -37,11 +37,17 @@ pub struct Module {
 
     /// A declaration that has been shadowed (overwritten) by this module.
     pub shadowed: Option<Box<Decl>>,
+
+    /// When a reference into this module does not exist, this declaration will
+    /// be instantiated instead.
+    pub infer_decl: Option<Box<Decl>>,
 }
 
-/// A struct containing information about a single declaration.
+/// A struct containing information about a single declaration
+/// within a PRQL module.
 #[derive(Debug, PartialEq, Default, Serialize, Deserialize, Clone)]
 pub struct Decl {
+    // TODO: make this plain usize, it is populated at creation anyway
     #[serde(skip_serializing_if = "Option::is_none")]
     pub declared_at: Option<usize>,
 
@@ -56,7 +62,7 @@ pub struct Decl {
     pub annotations: Vec<Annotation>,
 }
 
-/// The Declaration itself.
+/// Declaration kind.
 #[derive(Debug, PartialEq, Serialize, Deserialize, Clone, EnumAsInner)]
 pub enum DeclKind {
     /// A nested namespace
@@ -85,6 +91,11 @@ pub enum DeclKind {
 
     /// Equivalent to the declaration pointed to by the fully qualified ident
     Import(Ident),
+
+    /// A declaration that has not yet been resolved.
+    /// Created during the first pass of the AST, must not be present in
+    /// a fully resolved module structure.
+    Unresolved(StmtKind),
 }
 
 #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
@@ -163,6 +174,7 @@ impl Default for DeclKind {
     }
 }
 
+// TODO: convert to Decl::new
 impl From<DeclKind> for Decl {
     fn from(kind: DeclKind) -> Self {
         Decl {
@@ -199,6 +211,7 @@ impl std::fmt::Display for DeclKind {
             Self::Ty(arg0) => write!(f, "Ty: {}", write_ty(arg0)),
             Self::QueryDef(_) => write!(f, "QueryDef"),
             Self::Import(arg0) => write!(f, "Import {arg0}"),
+            Self::Unresolved(_) => write!(f, "Unresolved"),
         }
     }
 }

diff --git a/prqlc/prqlc/src/ir/pl/fold.rs b/prqlc/prqlc/src/ir/pl/fold.rs
@@ -277,13 +277,18 @@ pub fn fold_transform_kind<T: ?Sized + PlFold>(
 
 pub fn fold_func<T: ?Sized + PlFold>(fold: &mut T, func: Func) -> Result<Func> {
     Ok(Func {
+        name_hint: func.name_hint,
         body: Box::new(fold.fold_expr(*func.body)?),
         args: func
             .args
             .into_iter()
             .map(|item| fold.fold_expr(item))
             .try_collect()?,
-        ..func
+        return_ty: fold_type_opt(fold, func.return_ty)?,
+        params: fold_func_param(fold, func.params)?,
+        named_params: fold_func_param(fold, func.named_params)?,
+        generic_type_params: func.generic_type_params, // recurse into this too?
+        env: func.env,                                 // recurse into this too?
     })
 }
 
@@ -295,8 +300,9 @@ pub fn fold_func_param<T: ?Sized + PlFold>(
         .into_iter()
         .map(|param| {
             Ok(FuncParam {
+                name: param.name,
+                ty: fold_type_opt(fold, param.ty)?,
                 default_value: fold_optional_box(fold, param.default_value)?,
-                ..param
             })
         })
         .try_collect()

diff --git a/prqlc/prqlc/src/semantic/ast_expand.rs b/prqlc/prqlc/src/semantic/ast_expand.rs
@@ -9,6 +9,8 @@ use crate::ir::pl::{self, new_binop};
 use crate::semantic::{NS_THAT, NS_THIS};
 use crate::{Error, Result};
 
+use super::NS_LOCAL;
+
 /// An AST pass that maps AST to PL.
 pub fn expand_expr(expr: Expr) -> Result<pl::Expr> {
     let kind = match expr.kind {
@@ -306,7 +308,13 @@ fn restrict_exprs(exprs: Vec<pl::Expr>) -> Vec<Expr> {
 fn restrict_expr_kind(value: pl::ExprKind) -> ExprKind {
     match value {
         pl::ExprKind::Ident(v) => {
+            // HACK: remove the '_local' prefix
+            let skip_first = v.starts_with_part(NS_LOCAL);
+
             let mut parts = v.into_iter();
+            if skip_first {
+                parts.next();
+            }
             let mut base = Box::new(Expr::new(ExprKind::Ident(parts.next().unwrap())));
             for part in parts {
                 let field = IndirectionKind::Name(part);
@@ -508,6 +516,7 @@ fn restrict_decl(name: String, value: decl::Decl) -> Option<Stmt> {
         }
         decl::DeclKind::Column(id) => new_internal_stmt(name, format!("column.{id}")),
         decl::DeclKind::Infer(_) => new_internal_stmt(name, "infer".to_string()),
+        decl::DeclKind::Unresolved(_) => new_internal_stmt(name, "unresolved".to_string()),
 
         decl::DeclKind::Expr(mut expr) => StmtKind::VarDef(VarDef {
             kind: VarDefKind::Let,

diff --git a/prqlc/prqlc/src/semantic/lowering.rs b/prqlc/prqlc/src/semantic/lowering.rs
@@ -855,6 +855,7 @@ impl Lowerer {
                     );
                 }
             }
+
             pl::ExprKind::Literal(literal) => rq::ExprKind::Literal(literal),
 
             pl::ExprKind::SString(items) => {

diff --git a/prqlc/prqlc/src/semantic/mod.rs b/prqlc/prqlc/src/semantic/mod.rs
@@ -5,6 +5,7 @@ mod eval;
 mod lowering;
 mod module;
 pub mod reporting;
+mod resolve_decls;
 mod resolver;
 
 use self::resolver::Resolver;
@@ -14,7 +15,7 @@ pub use lowering::lower_to_ir;
 
 use crate::ast;
 use crate::ir::constant::ConstExpr;
-use crate::ir::decl::{Module, RootModule};
+use crate::ir::decl::RootModule;
 use crate::ir::pl::{self, Expr, ImportDef, ModuleDef, Stmt, StmtKind, TypeDef, VarDef};
 use crate::ir::rq::RelationalQuery;
 use crate::parser::is_mod_def_for;
@@ -42,15 +43,18 @@ pub fn resolve(mut module_tree: ast::ModuleDef, options: ResolverOptions) -> Res
     // expand AST into PL
     let root_module_def = ast_expand::expand_module_def(module_tree)?;
 
-    // init new root module
-    let mut root_module = RootModule {
-        module: Module::new_root(),
-        ..Default::default()
-    };
+    // init the module structure
+    let mut root_module = resolve_decls::init_module_tree(root_module_def);
+
+    // resolve name references between declarations
+    let resolution_order = resolve_decls::resolve_decl_refs(&mut root_module)?;
+
+    // resolve
     let mut resolver = Resolver::new(&mut root_module, options);
 
-    // resolve the module def into the root module
-    resolver.fold_statements(root_module_def.stmts)?;
+    for decl_fq in resolution_order {
+        resolver.resolve_decl(decl_fq)?;
+    }
 
     Ok(root_module)
 }
@@ -99,6 +103,7 @@ pub const NS_PARAM: &str = "_param";
 pub const NS_DEFAULT_DB: &str = "db";
 pub const NS_QUERY_DEF: &str = "prql";
 pub const NS_MAIN: &str = "main";
+pub const NS_LOCAL: &str = "_local";
 
 // refers to the containing module (direct parent)
 pub const NS_SELF: &str = "_self";
-Original file line number
+Diff line change
@@ Expand Up / @@ -855,6 +855,7 @@ impl Lowerer { @@
                         );
                     }
                 }
                 pl::ExprKind::Literal(literal) => rq::ExprKind::Literal(literal),
                 pl::ExprKind::SString(items) => {
@@ Expand Down @@