Add information about documentplacement in tokens

crates/filerunner/Cargo.toml

@@ -4,4 +4,6 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-interpreter =  { path = "../interpreter" }
+interpreter = { path = "../interpreter" }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }

crates/filerunner/src/lib.rs

@@ -1,6 +1,9 @@
 use std::fs;
 
-use interpreter::eval::{self, objects::Environment, EvaledProgram};
+use interpreter::{
+    eval::{self, objects::Environment, EvaledProgram},
+    parser::{ParsedProgram, Parser},
+};
 
 pub fn execute_file(filename: &str) -> Result<(), std::io::Error> {
     let file_content = fs::read_to_string(filename)?;
@@ -11,6 +14,24 @@ pub fn execute_file(filename: &str) -> Result<(), std::io::Error> {
     Ok(())
 }
 
+pub fn parse_file(filename: &str) -> Result<(), std::io::Error> {
+    let file_content = fs::read_to_string(filename)?;
+    let mut parser = Parser::new(&file_content);
+
+    let parsed_program = parser.parse_program();
+    match parsed_program {
+        ParsedProgram::ValidProgram(_) => println!("Program contained no errors"),
+        ParsedProgram::InvalidProgram(parse_errors) => {
+            eprintln!("Found parse errors:");
+            parse_errors.into_iter().for_each(|error| {
+                eprintln!("{error}");
+            });
+        }
+    }
+
+    Ok(())
+}
+
 fn handle_output(evaluated_output: EvaledProgram) {
     match evaluated_output {
         EvaledProgram::Valid(object) => println!("{object}"),

crates/filerunner/tests/error_message_test.rs

@@ -0,0 +1,53 @@
+use std::{fs, path::PathBuf, sync::Once};
+
+use interpreter::parser::{ParsedProgram, Parser};
+use tracing_subscriber::FmtSubscriber;
+
+static TRACING: Once = Once::new();
+pub fn setup_logger() {
+    let subscriber = FmtSubscriber::builder()
+        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+        .finish();
+
+    TRACING.call_once(|| {
+        tracing::subscriber::set_global_default(subscriber)
+            .expect("setting default subscriber failed");
+    })
+}
+
+#[test]
+fn test_error_message() {
+    setup_logger();
+
+    let mut test_file_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    test_file_path.push("tests/test_error.las");
+    let file_name = test_file_path.to_str().unwrap();
+
+    let file_content = fs::read_to_string(file_name).expect("Should be able to read string");
+
+    let mut parser = Parser::new(&file_content);
+    let program = parser.parse_program();
+
+    let first_expected_parse_error = "Error on line 2: expected binding to let statement\n";
+    let second_expected_parse_error = "Error on line 4: expected identifier in let statement\n";
+    match program {
+        ParsedProgram::InvalidProgram(parse_errors) => {
+            if parse_errors.len() != 2 {
+                for ele in parse_errors {
+                    println!("{ele}");
+                }
+                panic!("Expected 2 parse errors, but got the above ones instaed");
+            }
+
+            assert_eq!(
+                first_expected_parse_error,
+                parse_errors.first().unwrap().to_string()
+            );
+            assert_eq!(
+                second_expected_parse_error,
+                parse_errors.get(1).unwrap().to_string()
+            );
+        }
+        _ => panic!("Should return error"),
+    }
+}

crates/filerunner/tests/test_error.las

@@ -0,0 +1,5 @@
+let foo: "bar"
+let bar:  
+let hehe: "test"
+let : "test"
+let ok: "test"

crates/interpreter/src/eval.rs

@@ -1,9 +1,7 @@
 use eval_error::EvalError;
 use objects::{EnvReference, Object};
 
-use crate::parser::{
-    lexer::lexedtokens::LexedTokens, parse_errors::ParseError, ParsedProgram, Parser,
-};
+use crate::parser::{ParsedProgram, Parser, StatementError};
 
 mod array_evaluator;
 pub mod builtin;
@@ -14,15 +12,14 @@ pub mod objects;
 mod statement_evaluator;
 
 pub enum EvaledProgram {
-    ParseError(Vec<ParseError>),
+    ParseError(Vec<StatementError>),
     EvalError(EvalError),
     Valid(Object),
 }
 
 pub fn eval(input: &str, env: &mut EnvReference) -> EvaledProgram {
-    let lexed_tokens = LexedTokens::from(input);
-    let program = Parser::parse_tokens(lexed_tokens);
-
+    let mut parser = Parser::new(input);
+    let program = parser.parse_program();
     match program {
         ParsedProgram::InvalidProgram(parse_errors) => EvaledProgram::ParseError(parse_errors),
         ParsedProgram::ValidProgram(valid_program) => {

crates/interpreter/src/parser.rs

@@ -5,38 +5,46 @@ pub mod lexer;
 pub mod parse_errors;
 pub mod return_statement;
 
+use std::fmt::Display;
+
 use expressions::expression_statement;
+use parse_errors::ParseErrorKind;
 use tracing::{event, span, Level};
 
 use crate::{
     parser::assign_statement::AssignStatement,
     parser::ast::Statement,
-    parser::lexer::{lexedtokens::LexedTokens, token::Token},
+    parser::lexer::{lexedtokens::Lexer, token::TokenKind},
     parser::parse_errors::ParseError,
     parser::return_statement::ReturnStatement,
 };
 
-pub struct Parser {
-    pub tokens: LexedTokens,
+pub struct Parser<'a> {
+    pub lexer: Lexer<'a>,
 }
 
 pub enum ParsedProgram {
     ValidProgram(Vec<Statement>),
-    InvalidProgram(Vec<ParseError>),
+    InvalidProgram(Vec<StatementError>),
 }
 
-impl Parser {
-    pub fn parse_tokens(tokens: LexedTokens) -> ParsedProgram {
-        let mut parser = Parser { tokens };
+impl<'a> Parser<'a> {
+    pub fn new(input: &'a str) -> Self {
+        Parser {
+            lexer: Lexer::new(input),
+        }
+    }
+    pub fn parse_tokens(tokens: Lexer) -> ParsedProgram {
+        let mut parser = Parser { lexer: tokens };
 
         parser.parse_program()
     }
 
-    fn parse_program(&mut self) -> ParsedProgram {
+    pub fn parse_program(&mut self) -> ParsedProgram {
         let mut statements: Vec<Statement> = Vec::new();
-        let mut parse_errors: Vec<ParseError> = Vec::new();
+        let mut parse_errors: Vec<StatementError> = Vec::new();
 
-        while self.tokens.peek().is_some() {
+        while self.lexer.has_next() {
             let statement_span = span!(Level::DEBUG, "Statement");
             let _enter = statement_span.enter();
 
@@ -46,8 +54,9 @@ impl Parser {
                     statements.push(parsed_statement)
                 }
                 Err(parse_error) => {
-                    event!(Level::DEBUG, "Error parsing statement: {parse_error:?}");
-                    self.tokens.iterate_to_next_statement();
+                    // TODO: maybe it is possible to defer creating the error object
+                    // with the correct document location to here
+                    self.lexer.iterate_to_next_statement();
                     parse_errors.push(parse_error)
                 }
             };
@@ -60,12 +69,66 @@ impl Parser {
         ParsedProgram::ValidProgram(statements)
     }
 
-    fn parse_statement(&mut self) -> Result<Statement, ParseError> {
-        match self.tokens.peek() {
-            Some(Token::Return) => ReturnStatement::parse_return_statement(self),
-            Some(Token::Let) => AssignStatement::parse(self),
-            Some(_) => expression_statement::parse(self),
-            None => Err(ParseError::ExpectedToken),
+    fn parse_statement(&mut self) -> Result<Statement, StatementError> {
+        let current_token = self
+            .lexer
+            .expect_peek()
+            .expect("Function should be called after assertion");
+
+        event!(
+            Level::DEBUG,
+            "Parsing statement begining with token_kind: {:?}",
+            current_token
+        );
+        match current_token.token_kind {
+            TokenKind::Return => ReturnStatement::parse_return_statement(self),
+            TokenKind::Let => AssignStatement::parse(self),
+            _ => expression_statement::parse(self),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct StatementError {
+    parse_error: ParseError,
+    statement_type: StatementType,
+}
+
+// TODO: Investigate whether we can use a struct for statements with this
+// as the type, instead of an enum of statements
+#[derive(Debug)]
+pub enum StatementType {
+    Return,
+    Let,
+    Expression,
+}
+
+impl Display for StatementError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Error on line {}: ",
+            self.parse_error.token.location.line_number
+        )?;
+        match self.statement_type {
+            StatementType::Return => writeln!(f, "{}", self.parse_error),
+            StatementType::Let => match &self.parse_error.kind {
+                ParseErrorKind::NoPrefixExpression => {
+                    writeln!(f, "expected binding to let statement")
+                }
+                ParseErrorKind::UnexpectedToken(expected_token) => match expected_token {
+                    TokenKind::Ident(_) => {
+                        writeln!(f, "expected identifier in let statement")
+                    }
+                    not_handled => writeln!(
+                        f,
+                        "expected {} in variable declaration, got {}",
+                        expected_token, not_handled
+                    ),
+                },
+                _ => writeln!(f, "{}", self.parse_error),
+            },
+            StatementType::Expression => writeln!(f, "{}", self.parse_error),
         }
     }
 }