basic parser tests

src/parser/parser.rs

@@ -1,8 +1,21 @@
 use super::types::*;
 use crate::ast;
 
-fn parse_definition(tokens: &mut Tokens) -> Result<ast::Definition, Error> {
+pub fn parse<T>(tokens: &mut Tokens, parser: Parser<T>) -> Result<T, Error> {
-    if let Some(start) = tokens.next_if(Token::Let) {
+    if let Some(result) = parser(tokens)?
+        && tokens.peek().is_none()
+    {
+        Ok(result)
+    } else {
+        Err(Error::UnexpectedTokenForParser(
+            "parser".into(),
+            tokens.next(),
+        ))
+    }
+}
+
+fn parse_definition(tokens: &mut Tokens) -> ParseResult<ast::Definition> {
+    if let Some(_) = tokens.next_if(Token::Let) {
         if let Some(LocatedToken {
             token: Token::Identifier(identifier),
             ..
@@ -10,30 +23,38 @@ fn parse_definition(tokens: &mut Tokens) -> Result<ast::Definition, Error> {
         {
             if let Some(_eq) = tokens.next_if(Token::Equals) {
                 if let Some(expr) = parse_expr(tokens)? {
-                    Ok(ast::Definition {
+                    if let Some(_) = tokens.next_if(Token::Semicolon) {
-                        name: ast::Name(identifier),
+                        Ok(Some(ast::Definition {
-                        expr,
+                            name: ast::Name(identifier),
-                    })
+                            expr,
+                        }))
+                    } else {
+                        Err(Error::UnexpectedToken {
+                            expected: Token::Semicolon,
+                            got: tokens.next(),
+                        })
+                    }
                 } else {
-                    Err(Error::UnexpectedToken {
+                    Err(Error::UnexpectedTokenForParser(
-                        expected: Token::Equals,
+                        "expr".into(),
-                        got: start,
+                        tokens.next(),
-                    })
+                    ))
                 }
             } else {
                 Err(Error::UnexpectedToken {
                     expected: Token::Equals,
-                    got: start,
+                    got: tokens.next(),
                 })
             }
         } else {
             Err(Error::UnexpectedToken {
-                expected: Token::Equals,
+                expected: Token::Identifier("".into()),
-                got: start,
+                got: tokens.next(),
             })
         }
     } else {
-        Err(Error::UnexpectedEndOfInput)
+        println!("not let! {tokens:#?}");
+        Ok(None)
     }
 }
 
@@ -43,11 +64,11 @@ fn parse_expr(tokens: &mut Tokens) -> ParseResult<ast::Expr> {
 
 fn parse_simple_expr(tokens: &mut Tokens) -> ParseResult<ast::Expr> {
     tokens.one_of(vec![
-        |t| t.between(Token::OpenParen, Token::CloseParen, parse_expr),
-        parse_identifier,
         parse_number,
+        parse_identifier,
         parse_string,
         parse_bool,
+        |t| t.between(Token::OpenParen, Token::CloseParen, parse_expr),
     ])
 }
 
@@ -115,3 +136,22 @@ fn parse_bool(tokens: &mut Tokens) -> ParseResult<ast::Expr> {
         },
     ])
 }
+
+#[cfg(test)]
+mod tests {
+    use super::super::scanner::*;
+    use super::*;
+
+    #[test]
+    fn number() {
+        let program = "108".to_string();
+        let result = parse(&mut scan(program), parse_expr);
+        insta::assert_debug_snapshot!(result);
+    }
+    #[test]
+    fn let_number() {
+        let program = "let x = 108;".to_string();
+        let result = parse(&mut scan(program), parse_definition);
+        insta::assert_debug_snapshot!(result);
+    }
+}

src/parser/scanner.rs

@@ -3,7 +3,7 @@ use chumsky::text::Char;
 use log;
 use lyn::Scanner;
 
-fn scan(source: String) -> Vec<LocatedToken> {
+pub fn scan(source: String) -> Tokens {
     let mut scanner = Scanner::new(&source);
     let mut tokens = Vec::new();
 
@@ -153,7 +153,7 @@ fn scan(source: String) -> Vec<LocatedToken> {
         }
     }
 
-    tokens
+    tokens.into()
 }
 
 #[cfg(test)]
@@ -163,10 +163,12 @@ mod tests {
     #[test]
     fn let_number() {
         let program = "let x = 108;".to_string();
-        let result = scan(program)
+        let mut result = scan(program)
+            .0
             .into_iter()
             .map(|t| t.token)
             .collect::<Vec<_>>();
+        result.reverse();
         insta::assert_debug_snapshot!(result);
     }
 
@@ -177,10 +179,12 @@ let main = fn (x) {
   console.log(x);
 };"
         .to_string();
-        let result = scan(program)
+        let mut result = scan(program)
+            .0
             .into_iter()
             .map(|t| t.token)
             .collect::<Vec<_>>();
+        result.reverse();
         insta::assert_debug_snapshot!(result);
     }
 
@@ -206,10 +210,12 @@ let migrate = fn(state) {
 };
 "
         .to_string();
-        let result = scan(program)
+        let mut result = scan(program)
+            .0
             .into_iter()
             .map(|t| t.token)
             .collect::<Vec<_>>();
+        result.reverse();
         insta::assert_debug_snapshot!(result);
     }
     /*

src/parser/snapshots/ayin__parser__parser__tests__let_number.snap

@@ -0,0 +1,16 @@
+---
+source: src/parser/parser.rs
+expression: result
+---
+Ok(
+    Definition {
+        name: Name(
+            "x",
+        ),
+        expr: Value(
+            Int(
+                108,
+            ),
+        ),
+    },
+)

src/parser/snapshots/ayin__parser__parser__tests__number.snap

@@ -0,0 +1,11 @@
+---
+source: src/parser/parser.rs
+expression: result
+---
+Ok(
+    Value(
+        Int(
+            108,
+        ),
+    ),
+)

src/parser/types.rs

@@ -28,7 +28,15 @@ pub enum Token {
     Identifier(String),
 }
 
-pub struct Tokens(Vec<LocatedToken>);
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Tokens(pub Vec<LocatedToken>);
+
+impl From<Vec<LocatedToken>> for Tokens {
+    fn from(mut tokens: Vec<LocatedToken>) -> Tokens {
+        tokens.reverse();
+        Tokens(tokens)
+    }
+}
 
 impl Tokens {
     pub fn peek(&mut self) -> Option<&LocatedToken> {
@@ -38,21 +46,30 @@ impl Tokens {
         self.0.pop()
     }
     pub fn next_if(&mut self, token: Token) -> Option<LocatedToken> {
-        self.0.pop_if(|t| match (&t.token, &token) {
+        self.0.pop_if(|t| match (t.token.clone(), token) {
             (a, b) if a == b => true,
             (Token::Identifier(_), Token::Identifier(_)) => true,
             (Token::Number(_), Token::Number(_)) => true,
             (Token::String(_), Token::String(_)) => true,
-            _ => false,
+            x => {
+                println!("next_if fail: {x:#?}");
+                false
+            }
         })
     }
     pub fn between<T>(&mut self, start: Token, end: Token, parser: Parser<T>) -> ParseResult<T> {
-        let _start = self.next_if(start).ok_or(Error::UnexpectedEndOfInput)?;
+        if let Some(_start) = self.next_if(start) {
-        if let Some(result) = parser(self)? {
+            if let Some(result) = parser(self)? {
-            let _end = self.next_if(end).ok_or(Error::UnexpectedEndOfInput)?;
+                let _end = self.next_if(end).ok_or(Error::UnexpectedEndOfInput)?;
-            Ok(Some(result))
+                Ok(Some(result))
+            } else {
+                Err(Error::UnexpectedTokenForParser(
+                    "between".into(),
+                    self.next(),
+                ))
+            }
         } else {
-            Err(Error::UnexpectedTokenForParser)
+            Ok(None)
         }
     }
     pub fn one_of<T>(&mut self, parsers: Vec<Parser<T>>) -> ParseResult<T> {
@@ -74,7 +91,10 @@ impl Tokens {
             if let Some(t2) = parser2(self)? {
                 Ok(Some((t1, t2)))
             } else {
-                Err(Error::UnexpectedTokenForParser)
+                Err(Error::UnexpectedTokenForParser(
+                    "sep_by".into(),
+                    self.next(),
+                ))
             }
         } else {
             Ok(None)
@@ -92,8 +112,12 @@ impl Tokens {
 pub type Parser<T> = fn(&mut Tokens) -> ParseResult<T>;
 pub type ParseResult<T> = Result<Option<T>, Error>;
 
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Error {
-    UnexpectedToken { expected: Token, got: LocatedToken },
+    UnexpectedToken {
+        expected: Token,
+        got: Option<LocatedToken>,
+    },
     UnexpectedEndOfInput,
-    UnexpectedTokenForParser,
+    UnexpectedTokenForParser(String, Option<LocatedToken>),
 }