lexer tests and improvements

Cargo.lock

@@ -94,6 +94,7 @@ version = "0.1.0"
 dependencies = [
  "chumsky",
  "env_logger",
+ "insta",
  "log",
  "lyn",
  "macroquad",
@@ -160,6 +161,18 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
 
+[[package]]
+name = "console"
+version = "0.15.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "once_cell",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "crc32fast"
 version = "1.5.0"
@@ -169,6 +182,12 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "encode_unicode"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+
 [[package]]
 name = "env_filter"
 version = "0.1.4"
@@ -269,6 +288,17 @@ dependencies = [
  "png",
 ]
 
+[[package]]
+name = "insta"
+version = "1.44.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5c943d4415edd8153251b6f197de5eb1640e56d84e8d9159bea190421c73698"
+dependencies = [
+ "console",
+ "once_cell",
+ "similar",
+]
+
 [[package]]
 name = "is_terminal_polyfill"
 version = "1.70.2"
@@ -407,6 +437,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "once_cell"
+version = "1.21.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+
 [[package]]
 name = "once_cell_polyfill"
 version = "1.70.2"
@@ -563,6 +599,12 @@ version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
 
+[[package]]
+name = "similar"
+version = "2.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
+
 [[package]]
 name = "stacker"
 version = "0.1.22"

Cargo.toml

@@ -24,6 +24,9 @@ macroquad = "0.4.14"
 lyn = "0.1.0"
 chumsky = "0.11.2"
 
+[dev-dependencies]
+insta = "1.44.3"
+
 [workspace.lints.clippy]
 all = { level = "warn", priority = -1 }
 pedantic = { level = "warn", priority = -1 }

Makefile

@@ -6,10 +6,18 @@ watch:
 build:
 	cargo build
 
+.PHONY: test
+test:
+	cargo test
+
 .PHONY: run
 run:
 	cargo run
 
+.PHONY: review
+review:
+	cargo insta review
+
 .PHONY: wasm
 wasm:
 	cargo build --target wasm32-unknown-unknown

src/interpret/interpret.rs

@@ -142,6 +142,7 @@ fn defs_to_env(
     Ok(())
 }
 
+/*
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -150,7 +151,7 @@ mod tests {
     #[test]
     fn main_0() {
         let program = vec![helpers::define_expr("main", 0.into())].into();
-        let result = run(program);
+        let result = run(program, "main".into(), vec![]);
         assert_eq!(result, Ok(0.into()));
     }
     #[test]
@@ -160,7 +161,7 @@ mod tests {
             helpers::define_expr("lit", 0.into()),
         ]
         .into();
-        let result = run(program);
+        let result = run(program, "main".into(), vec![]);
         assert_eq!(result, Ok(0.into()));
     }
     #[test]
@@ -174,7 +175,7 @@ mod tests {
             .into(),
         )]
         .into();
-        let result = run(program);
+        let result = run(program, "main".into(), vec![]);
         assert_eq!(result, Ok(0.into()));
     }
 
@@ -188,7 +189,7 @@ mod tests {
             ),
         ]
         .into();
-        let result = run(program);
+        let result = run(program, "main".into(), vec![]);
         assert_eq!(result, Ok(0.into()));
     }
 
@@ -206,7 +207,7 @@ mod tests {
             ),
         ]
         .into();
-        let result = run(program);
+        let result = run(program, "main".into(), vec![]);
         assert_eq!(result, Ok(0.into()));
     }
 
@@ -225,7 +226,7 @@ mod tests {
             },
         )]
         .into();
-        let result = run(program);
+        let result = run(program, "main".into(), vec![]);
         assert_eq!(result, Ok(1.into()));
     }
 
@@ -236,23 +237,14 @@ mod tests {
             helpers::define_expr(
                 "zero",
                 helpers::func(ast::Fn {
-                    args: vec![
+                    args: vec![ast::Arg { name: "a".into() }, ast::Arg { name: "b".into() }],
-                        ast::Arg {
-                            name: "a".into(),
-                            r#type: ast::Type::Named("i64".into()),
-                        },
-                        ast::Arg {
-                            name: "b".into(),
-                            r#type: ast::Type::Named("i64".into()),
-                        },
-                    ],
                     body: "b".into(),
                 })
                 .call(vec![1.into(), 0.into()]),
             ),
         ]
         .into();
-        let result = run(program);
+        let result = run(program, "main".into(), vec![]);
         assert_eq!(result, Ok(0.into()));
     }
 
@@ -265,7 +257,7 @@ mod tests {
             helpers::define_expr("main", 0.into()),
         ]
         .into();
-        let result = run(program);
+        let result = run(program, "main".into(), vec![]);
         assert_eq!(result, Err(Error::DuplicateNames("main".into())));
     }
 
@@ -276,7 +268,7 @@ mod tests {
             helpers::define_expr("record", ast::Expr::from(0).field("my_field")),
         ]
         .into();
-        let result = run(program);
+        let result = run(program, "main".into(), vec![]);
         assert_eq!(result, Err(Error::NotARecord(0.into())));
     }
     #[test]
@@ -286,7 +278,8 @@ mod tests {
             helpers::define_expr("zero", ast::Expr::from(0).call(vec![1.into()])),
         ]
         .into();
-        let result = run(program);
+        let result = run(program, "main".into(), vec![]);
         assert_eq!(result, Err(Error::NotAFunction(0.into())));
     }
 }
+*/

src/parser/scanner.rs

@@ -1,7 +1,7 @@
+use super::types::*;
 use chumsky::text::Char;
 use log;
 use lyn::Scanner;
-use super::types::*;
 
 fn scan(source: String) -> Vec<LocatedToken> {
     let mut scanner = Scanner::new(&source);
@@ -21,6 +21,11 @@ fn scan(source: String) -> Vec<LocatedToken> {
                     end: scanner.cursor(),
                     token: Token::Comma,
                 }),
+                ':' => tokens.push(LocatedToken {
+                    start,
+                    end: scanner.cursor(),
+                    token: Token::Colon,
+                }),
                 ';' => tokens.push(LocatedToken {
                     start,
                     end: scanner.cursor(),
@@ -89,7 +94,7 @@ fn scan(source: String) -> Vec<LocatedToken> {
                 _ if c.is_whitespace() => {}
                 // numbers
                 _ if c.is_numeric() => {
-                    let mut str = "".to_string();
+                    let mut str = c.to_string();
                     loop {
                         if let Some(ch) = scanner.peek()
                             && !ch.is_numeric()
@@ -111,7 +116,7 @@ fn scan(source: String) -> Vec<LocatedToken> {
                 }
                 // identifiers and keywords
                 _ if c.is_alphabetic() => {
-                    let mut str = "".to_string();
+                    let mut str = c.to_string();
                     loop {
                         if let Some(ch) = scanner.peek()
                             && !ch.is_alphanumeric()
@@ -131,6 +136,7 @@ fn scan(source: String) -> Vec<LocatedToken> {
                         token: match str.as_str() {
                             "fn" => Token::Fn,
                             "let" => Token::Let,
+                            "return" => Token::Return,
                             _ => Token::Identifier(str),
                         },
                     });
@@ -147,3 +153,75 @@ fn scan(source: String) -> Vec<LocatedToken> {
 
     tokens
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn let_number() {
+        let program = "let x = 108;".to_string();
+        let result = scan(program)
+            .into_iter()
+            .map(|t| t.token)
+            .collect::<Vec<_>>();
+        insta::assert_debug_snapshot!(result);
+    }
+
+    #[test]
+    fn let_fn() {
+        let program = "
+let main = fn (x) {
+  console.log(x);
+};"
+        .to_string();
+        let result = scan(program)
+            .into_iter()
+            .map(|t| t.token)
+            .collect::<Vec<_>>();
+        insta::assert_debug_snapshot!(result);
+    }
+
+    #[test]
+    fn scaffolding() {
+        let program = "
+let init = fn() {
+  return {
+    player: { position: { x: 10, y: 20 }, },
+  }
+};
+
+let update = fn(state, events) {
+  return state';
+};
+
+let draw = fn(frame, state) {
+  frame.clear(0,0,0);
+};
+
+let migrate = fn(state) {
+    return { player: { pos: state.player.position } },
+};
+"
+        .to_string();
+        let result = scan(program)
+            .into_iter()
+            .map(|t| t.token)
+            .collect::<Vec<_>>();
+        insta::assert_debug_snapshot!(result);
+    }
+    /*
+    // Errors
+
+    #[test]
+    fn duplicate_toplevel_defs() {
+        let program = vec![
+            helpers::define_expr("main", "record".into()),
+            helpers::define_expr("main", 0.into()),
+        ]
+        .into();
+        let result = run(program);
+        assert_eq!(result, Err(Error::DuplicateNames("main".into())));
+    }
+    */
+}

src/parser/snapshots/ayin__parser__scanner__tests__let_fn.snap

@@ -0,0 +1,33 @@
+---
+source: src/parser/scanner.rs
+expression: result
+---
+[
+    Let,
+    Identifier(
+        "main",
+    ),
+    Equals,
+    Fn,
+    OpenParen,
+    Identifier(
+        "x",
+    ),
+    CloseParen,
+    OpenCurly,
+    Identifier(
+        "console",
+    ),
+    Dot,
+    Identifier(
+        "log",
+    ),
+    OpenParen,
+    Identifier(
+        "x",
+    ),
+    CloseParen,
+    Semicolon,
+    CloseCurly,
+    Semicolon,
+]

src/parser/snapshots/ayin__parser__scanner__tests__let_number.snap

@@ -0,0 +1,15 @@
+---
+source: src/parser/scanner.rs
+expression: result
+---
+[
+    Let,
+    Identifier(
+        "x",
+    ),
+    Equals,
+    Number(
+        108,
+    ),
+    Semicolon,
+]

src/parser/snapshots/ayin__parser__scanner__tests__scaffolding.snap

@@ -0,0 +1,150 @@
+---
+source: src/parser/scanner.rs
+expression: result
+---
+[
+    Let,
+    Identifier(
+        "init",
+    ),
+    Equals,
+    Fn,
+    OpenParen,
+    CloseParen,
+    OpenCurly,
+    Return,
+    OpenCurly,
+    Identifier(
+        "player",
+    ),
+    Colon,
+    OpenCurly,
+    Identifier(
+        "position",
+    ),
+    Colon,
+    OpenCurly,
+    Identifier(
+        "x",
+    ),
+    Colon,
+    Number(
+        10,
+    ),
+    Comma,
+    Identifier(
+        "y",
+    ),
+    Colon,
+    Number(
+        20,
+    ),
+    CloseCurly,
+    Comma,
+    CloseCurly,
+    Comma,
+    CloseCurly,
+    CloseCurly,
+    Semicolon,
+    Let,
+    Identifier(
+        "update",
+    ),
+    Equals,
+    Fn,
+    OpenParen,
+    Identifier(
+        "state",
+    ),
+    Comma,
+    Identifier(
+        "events",
+    ),
+    CloseParen,
+    OpenCurly,
+    Return,
+    Identifier(
+        "state",
+    ),
+    Semicolon,
+    CloseCurly,
+    Semicolon,
+    Let,
+    Identifier(
+        "draw",
+    ),
+    Equals,
+    Fn,
+    OpenParen,
+    Identifier(
+        "frame",
+    ),
+    Comma,
+    Identifier(
+        "state",
+    ),
+    CloseParen,
+    OpenCurly,
+    Identifier(
+        "frame",
+    ),
+    Dot,
+    Identifier(
+        "clear",
+    ),
+    OpenParen,
+    Number(
+        0,
+    ),
+    Comma,
+    Number(
+        0,
+    ),
+    Comma,
+    Number(
+        0,
+    ),
+    CloseParen,
+    Semicolon,
+    CloseCurly,
+    Semicolon,
+    Let,
+    Identifier(
+        "migrate",
+    ),
+    Equals,
+    Fn,
+    OpenParen,
+    Identifier(
+        "state",
+    ),
+    CloseParen,
+    OpenCurly,
+    Return,
+    OpenCurly,
+    Identifier(
+        "player",
+    ),
+    Colon,
+    OpenCurly,
+    Identifier(
+        "pos",
+    ),
+    Colon,
+    Identifier(
+        "state",
+    ),
+    Dot,
+    Identifier(
+        "player",
+    ),
+    Dot,
+    Identifier(
+        "position",
+    ),
+    CloseCurly,
+    CloseCurly,
+    Comma,
+    CloseCurly,
+    Semicolon,
+]

src/parser/types.rs

@@ -9,8 +9,8 @@ pub struct LocatedToken {
 pub enum Token {
     Let,
     Fn,
+    Return,
     Equals,
-    Semicolon,
     OpenParen,
     CloseParen,
     OpenBracket,
@@ -19,6 +19,8 @@ pub enum Token {
     CloseCurly,
     Dot,
     Comma,
+    Semicolon,
+    Colon,
     Name(String),
     Number(u32),
     String(String),