me/ayin
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lexer tests and improvements

Browse source
This commit is contained in:
me 2025-12-15 00:40:01 +02:00
parent 6a2627ada6
commit 26924a17ae
9 changed files with 348 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
Cargo.lock generated
View file

@ -94,6 +94,7 @@ version = "0.1.0"
dependencies = [
"chumsky",
"env_logger",
"insta",
"log",
"lyn",
"macroquad",
@ -160,6 +161,18 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
[[package]]
name = "console"
version = "0.15.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
dependencies = [
"encode_unicode",
"libc",
"once_cell",
"windows-sys 0.59.0",
]
[[package]]
name = "crc32fast"
version = "1.5.0"
@ -169,6 +182,12 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "encode_unicode"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
[[package]]
name = "env_filter"
version = "0.1.4"
@ -269,6 +288,17 @@ dependencies = [
"png",
]
[[package]]
name = "insta"
version = "1.44.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5c943d4415edd8153251b6f197de5eb1640e56d84e8d9159bea190421c73698"
dependencies = [
"console",
"once_cell",
"similar",
]
[[package]]
name = "is_terminal_polyfill"
version = "1.70.2"
@ -407,6 +437,12 @@ dependencies = [
"memchr",
]
[[package]]
name = "once_cell"
version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "once_cell_polyfill"
version = "1.70.2"
@ -563,6 +599,12 @@ version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
[[package]]
name = "similar"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
[[package]]
name = "stacker"
version = "0.1.22"

3
Cargo.toml
View file

@ -24,6 +24,9 @@ macroquad = "0.4.14"
lyn = "0.1.0"
chumsky = "0.11.2"
[dev-dependencies]
insta = "1.44.3"
[workspace.lints.clippy]
all = { level = "warn", priority = -1 }
pedantic = { level = "warn", priority = -1 }

8
Makefile
View file

@ -6,10 +6,18 @@ watch:
build:
cargo build
.PHONY: test
test:
cargo test
.PHONY: run
run:
cargo run
.PHONY: review
review:
cargo insta review
.PHONY: wasm
wasm:
cargo build --target wasm32-unknown-unknown

33
src/interpret/interpret.rs
View file

@ -142,6 +142,7 @@ fn defs_to_env(
Ok(())
}
/*
#[cfg(test)]
mod tests {
use super::*;
@ -150,7 +151,7 @@ mod tests {
#[test]
fn main_0() {
let program = vec![helpers::define_expr("main", 0.into())].into();
let result = run(program);
let result = run(program, "main".into(), vec![]);
assert_eq!(result, Ok(0.into()));
}
#[test]
@ -160,7 +161,7 @@ mod tests {
helpers::define_expr("lit", 0.into()),
]
.into();
let result = run(program);
let result = run(program, "main".into(), vec![]);
assert_eq!(result, Ok(0.into()));
}
#[test]
@ -174,7 +175,7 @@ mod tests {
.into(),
)]
.into();
let result = run(program);
let result = run(program, "main".into(), vec![]);
assert_eq!(result, Ok(0.into()));
}
@ -188,7 +189,7 @@ mod tests {
),
]
.into();
let result = run(program);
let result = run(program, "main".into(), vec![]);
assert_eq!(result, Ok(0.into()));
}
@ -206,7 +207,7 @@ mod tests {
),
]
.into();
let result = run(program);
let result = run(program, "main".into(), vec![]);
assert_eq!(result, Ok(0.into()));
}
@ -225,7 +226,7 @@ mod tests {
},
)]
.into();
let result = run(program);
let result = run(program, "main".into(), vec![]);
assert_eq!(result, Ok(1.into()));
}
@ -236,23 +237,14 @@ mod tests {
helpers::define_expr(
"zero",
helpers::func(ast::Fn {
args: vec![
ast::Arg {
name: "a".into(),
r#type: ast::Type::Named("i64".into()),
},
ast::Arg {
name: "b".into(),
r#type: ast::Type::Named("i64".into()),
},
],
args: vec![ast::Arg { name: "a".into() }, ast::Arg { name: "b".into() }],
body: "b".into(),
})
.call(vec![1.into(), 0.into()]),
),
]
.into();
let result = run(program);
let result = run(program, "main".into(), vec![]);
assert_eq!(result, Ok(0.into()));
}
@ -265,7 +257,7 @@ mod tests {
helpers::define_expr("main", 0.into()),
]
.into();
let result = run(program);
let result = run(program, "main".into(), vec![]);
assert_eq!(result, Err(Error::DuplicateNames("main".into())));
}
@ -276,7 +268,7 @@ mod tests {
helpers::define_expr("record", ast::Expr::from(0).field("my_field")),
]
.into();
let result = run(program);
let result = run(program, "main".into(), vec![]);
assert_eq!(result, Err(Error::NotARecord(0.into())));
}
#[test]
@ -286,7 +278,8 @@ mod tests {
helpers::define_expr("zero", ast::Expr::from(0).call(vec![1.into()])),
]
.into();
let result = run(program);
let result = run(program, "main".into(), vec![]);
assert_eq!(result, Err(Error::NotAFunction(0.into())));
}
}
*/

84
src/parser/scanner.rs
View file

@ -1,7 +1,7 @@
use super::types::*;
use chumsky::text::Char;
use log;
use lyn::Scanner;
use super::types::*;
fn scan(source: String) -> Vec<LocatedToken> {
let mut scanner = Scanner::new(&source);
@ -21,6 +21,11 @@ fn scan(source: String) -> Vec<LocatedToken> {
end: scanner.cursor(),
token: Token::Comma,
}),
':' => tokens.push(LocatedToken {
start,
end: scanner.cursor(),
token: Token::Colon,
}),
';' => tokens.push(LocatedToken {
start,
end: scanner.cursor(),
@ -89,7 +94,7 @@ fn scan(source: String) -> Vec<LocatedToken> {
_ if c.is_whitespace() => {}
// numbers
_ if c.is_numeric() => {
let mut str = "".to_string();
let mut str = c.to_string();
loop {
if let Some(ch) = scanner.peek()
&& !ch.is_numeric()
@ -111,7 +116,7 @@ fn scan(source: String) -> Vec<LocatedToken> {
}
// identifiers and keywords
_ if c.is_alphabetic() => {
let mut str = "".to_string();
let mut str = c.to_string();
loop {
if let Some(ch) = scanner.peek()
&& !ch.is_alphanumeric()
@ -131,6 +136,7 @@ fn scan(source: String) -> Vec<LocatedToken> {
token: match str.as_str() {
"fn" => Token::Fn,
"let" => Token::Let,
"return" => Token::Return,
_ => Token::Identifier(str),
},
});
@ -147,3 +153,75 @@ fn scan(source: String) -> Vec<LocatedToken> {
tokens
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn let_number() {
let program = "let x = 108;".to_string();
let result = scan(program)
.into_iter()
.map(|t| t.token)
.collect::<Vec<_>>();
insta::assert_debug_snapshot!(result);
}
#[test]
fn let_fn() {
let program = "
let main = fn (x) {
console.log(x);
};"
.to_string();
let result = scan(program)
.into_iter()
.map(|t| t.token)
.collect::<Vec<_>>();
insta::assert_debug_snapshot!(result);
}
#[test]
fn scaffolding() {
let program = "
let init = fn() {
return {
player: { position: { x: 10, y: 20 }, },
}
};
let update = fn(state, events) {
return state';
};
let draw = fn(frame, state) {
frame.clear(0,0,0);
};
let migrate = fn(state) {
return { player: { pos: state.player.position } },
};
"
.to_string();
let result = scan(program)
.into_iter()
.map(|t| t.token)
.collect::<Vec<_>>();
insta::assert_debug_snapshot!(result);
}
/*
// Errors
#[test]
fn duplicate_toplevel_defs() {
let program = vec![
helpers::define_expr("main", "record".into()),
helpers::define_expr("main", 0.into()),
]
.into();
let result = run(program);
assert_eq!(result, Err(Error::DuplicateNames("main".into())));
}
*/
}

33
src/parser/snapshots/ayin__parser__scanner__tests__let_fn.snap Normal file
View file

@ -0,0 +1,33 @@
---
source: src/parser/scanner.rs
expression: result
---
[
Let,
Identifier(
"main",
),
Equals,
Fn,
OpenParen,
Identifier(
"x",
),
CloseParen,
OpenCurly,
Identifier(
"console",
),
Dot,
Identifier(
"log",
),
OpenParen,
Identifier(
"x",
),
CloseParen,
Semicolon,
CloseCurly,
Semicolon,
]

15
src/parser/snapshots/ayin__parser__scanner__tests__let_number.snap Normal file
View file

@ -0,0 +1,15 @@
---
source: src/parser/scanner.rs
expression: result
---
[
Let,
Identifier(
"x",
),
Equals,
Number(
108,
),
Semicolon,
]

150
src/parser/snapshots/ayin__parser__scanner__tests__scaffolding.snap Normal file
View file

@ -0,0 +1,150 @@
---
source: src/parser/scanner.rs
expression: result
---
[
Let,
Identifier(
"init",
),
Equals,
Fn,
OpenParen,
CloseParen,
OpenCurly,
Return,
OpenCurly,
Identifier(
"player",
),
Colon,
OpenCurly,
Identifier(
"position",
),
Colon,
OpenCurly,
Identifier(
"x",
),
Colon,
Number(
10,
),
Comma,
Identifier(
"y",
),
Colon,
Number(
20,
),
CloseCurly,
Comma,
CloseCurly,
Comma,
CloseCurly,
CloseCurly,
Semicolon,
Let,
Identifier(
"update",
),
Equals,
Fn,
OpenParen,
Identifier(
"state",
),
Comma,
Identifier(
"events",
),
CloseParen,
OpenCurly,
Return,
Identifier(
"state",
),
Semicolon,
CloseCurly,
Semicolon,
Let,
Identifier(
"draw",
),
Equals,
Fn,
OpenParen,
Identifier(
"frame",
),
Comma,
Identifier(
"state",
),
CloseParen,
OpenCurly,
Identifier(
"frame",
),
Dot,
Identifier(
"clear",
),
OpenParen,
Number(
0,
),
Comma,
Number(
0,
),
Comma,
Number(
0,
),
CloseParen,
Semicolon,
CloseCurly,
Semicolon,
Let,
Identifier(
"migrate",
),
Equals,
Fn,
OpenParen,
Identifier(
"state",
),
CloseParen,
OpenCurly,
Return,
OpenCurly,
Identifier(
"player",
),
Colon,
OpenCurly,
Identifier(
"pos",
),
Colon,
Identifier(
"state",
),
Dot,
Identifier(
"player",
),
Dot,
Identifier(
"position",
),
CloseCurly,
CloseCurly,
Comma,
CloseCurly,
Semicolon,
]

4
src/parser/types.rs
View file

@ -9,8 +9,8 @@ pub struct LocatedToken {
pub enum Token {
Let,
Fn,
Return,
Equals,
Semicolon,
OpenParen,
CloseParen,
OpenBracket,
@ -19,6 +19,8 @@ pub enum Token {
CloseCurly,
Dot,
Comma,
Semicolon,
Colon,
Name(String),
Number(u32),
String(String),