me/ayin
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fn parser

Browse source
This commit is contained in:
me 2025-12-16 15:08:02 +02:00
parent 31c70babc9
commit f5a41803e9
4 changed files with 122 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
src/parser/parser.rs
View file

@ -15,15 +15,12 @@ pub fn parse<T>(tokens: &mut Tokens, parser: Parser<T>) -> Result<T, Error> {
}
fn parse_definition(tokens: &mut Tokens) -> ParseResult<ast::Definition> {
if let Some(_) = tokens.next_if(Token::Let) {
if let Some(identifier) = parse_identifier_(tokens)? {
if let Some(_eq) = tokens.next_if(Token::Equals) {
if let Some(_) = tokens.next_if(&Token::Let) {
if let Some(name) = parse_identifier_name(tokens)? {
if let Some(_eq) = tokens.next_if(&Token::Equals) {
if let Some(expr) = parse_expr(tokens)? {
if let Some(_) = tokens.next_if(Token::Semicolon) {
Ok(Some(ast::Definition {
name: ast::Name(identifier),
expr,
}))
if let Some(_) = tokens.next_if(&Token::Semicolon) {
Ok(Some(ast::Definition { name, expr }))
} else {
Err(Error::UnexpectedToken {
expected: Token::Semicolon,
@ -63,15 +60,44 @@ fn parse_simple_expr(tokens: &mut Tokens) -> ParseResult<ast::Expr> {
parse_identifier,
parse_string,
parse_bool,
|t| t.between(Token::OpenParen, Token::CloseParen, parse_expr),
parse_fn,
|t| t.between(&Token::OpenParen, &Token::CloseParen, parse_expr),
])
}
fn parse_fn(tokens: &mut Tokens) -> ParseResult<ast::Expr> {
if let Some(_) = tokens.next_if(&Token::Fn) {
if let Some(args) = parse_args(tokens)? {
if let Some(body) = tokens.between(&Token::OpenCurly, &Token::CloseCurly, parse_expr)? {
Ok(Some(ast::Expr::Func(Box::new(ast::Fn { args, body }))))
} else {
Err(Error::UnexpectedTokenForParser(
"fn_body".into(),
tokens.next(),
))
}
} else {
Err(Error::UnexpectedTokenForParser(
"fn_args".into(),
tokens.next(),
))
}
} else {
Ok(None)
}
}
fn parse_args(tokens: &mut Tokens) -> ParseResult<Vec<ast::Arg>> {
tokens.between(&Token::OpenParen, &Token::CloseParen, |ts| {
ts.many_sep_by(&Token::Comma, parse_identifier_arg)
})
}
fn parse_string(tokens: &mut Tokens) -> ParseResult<ast::Expr> {
if let Some(LocatedToken {
token: Token::String(string),
..
}) = tokens.next_if(Token::String("".into()))
}) = tokens.next_if(&Token::String("".into()))
{
Ok(Some(ast::Expr::Value(ast::Value::String(string))))
} else {
@ -83,7 +109,7 @@ fn parse_number(tokens: &mut Tokens) -> ParseResult<ast::Expr> {
if let Some(LocatedToken {
token: Token::Number(number),
..
}) = tokens.next_if(Token::Number(0))
}) = tokens.next_if(&Token::Number(0))
{
Ok(Some(ast::Expr::Value(ast::Value::Int(number.into()))))
} else {
@ -92,16 +118,20 @@ fn parse_number(tokens: &mut Tokens) -> ParseResult<ast::Expr> {
}
fn parse_identifier(tokens: &mut Tokens) -> ParseResult<ast::Expr> {
Ok(parse_identifier_(tokens)?.map(|string| ast::Expr::Var(ast::Name(string))))
Ok(parse_identifier_name(tokens)?.map(|name| ast::Expr::Var(name)))
}
fn parse_identifier_(tokens: &mut Tokens) -> ParseResult<String> {
fn parse_identifier_arg(tokens: &mut Tokens) -> ParseResult<ast::Arg> {
Ok(parse_identifier_name(tokens)?.map(|name| ast::Arg { name }))
}
fn parse_identifier_name(tokens: &mut Tokens) -> ParseResult<ast::Name> {
if let Some(LocatedToken {
token: Token::Identifier(string),
..
}) = tokens.next_if(Token::Identifier("".into()))
}) = tokens.next_if(&Token::Identifier("".into()))
{
Ok(Some(string))
Ok(Some(ast::Name(string)))
} else {
Ok(None)
}
@ -111,7 +141,7 @@ fn parse_bool(tokens: &mut Tokens) -> ParseResult<ast::Expr> {
tokens.one_of(vec![
|t| {
Ok({
if let Some(_) = t.next_if(Token::True) {
if let Some(_) = t.next_if(&Token::True) {
Some(ast::Expr::Value(ast::Value::Boolean(true)))
} else {
None
@ -120,7 +150,7 @@ fn parse_bool(tokens: &mut Tokens) -> ParseResult<ast::Expr> {
},
|t| {
Ok({
if let Some(_) = t.next_if(Token::False) {
if let Some(_) = t.next_if(&Token::False) {
Some(ast::Expr::Value(ast::Value::Boolean(false)))
} else {
None
@ -148,6 +178,12 @@ mod tests {
insta::assert_debug_snapshot!(result);
}
#[test]
fn paren_fn() {
let program = "fn(a1, boco, c_c) { 108 }".to_string();
let result = parse(&mut scan(program), parse_expr);
insta::assert_debug_snapshot!(result);
}
#[test]
fn let_number() {
let program = "let x = 108;".to_string();
let result = parse(&mut scan(program), parse_definition);

3
src/parser/scanner.rs
View file

@ -115,11 +115,12 @@ pub fn scan(source: String) -> Tokens {
});
}
// identifiers and keywords
_ if c.is_alphabetic() => {
_ if c.is_alphabetic() || *c == '_' => {
let mut str = c.to_string();
loop {
if let Some(ch) = scanner.peek()
&& !ch.is_alphanumeric()
&& *ch != '_'
{
break;
}

32
src/parser/snapshots/ayin__parser__parser__tests__paren_fn.snap Normal file
View file

@ -0,0 +1,32 @@
---
source: src/parser/parser.rs
expression: result
---
Ok(
Func(
Fn {
args: [
Arg {
name: Name(
"a1",
),
},
Arg {
name: Name(
"boco",
),
},
Arg {
name: Name(
"c_c",
),
},
],
body: Value(
Int(
108,
),
),
},
),
)

42
src/parser/types.rs
View file

@ -45,19 +45,19 @@ impl Tokens {
pub fn next(&mut self) -> Option<LocatedToken> {
self.0.pop()
}
pub fn next_if(&mut self, token: Token) -> Option<LocatedToken> {
pub fn next_if(&mut self, token: &Token) -> Option<LocatedToken> {
self.0.pop_if(|t| match (t.token.clone(), token) {
(a, b) if a == b => true,
(a, b) if a == *b => true,
(Token::Identifier(_), Token::Identifier(_)) => true,
(Token::Number(_), Token::Number(_)) => true,
(Token::String(_), Token::String(_)) => true,
x => false,
_ => false,
})
}
pub fn between<T>(&mut self, start: Token, end: Token, parser: Parser<T>) -> ParseResult<T> {
pub fn between<T>(&mut self, start: &Token, end: &Token, parser: Parser<T>) -> ParseResult<T> {
if let Some(_start) = self.next_if(start) {
if let Some(result) = parser(self)? {
let _end = self.next_if(end).ok_or(Error::UnexpectedEndOfInput)?;
if let Some(_end) = self.next_if(end) {
Ok(Some(result))
} else {
Err(Error::UnexpectedTokenForParser(
@ -65,6 +65,12 @@ impl Tokens {
self.next(),
))
}
} else {
Err(Error::UnexpectedTokenForParser(
"between".into(),
self.next(),
))
}
} else {
Ok(None)
}
@ -79,12 +85,14 @@ impl Tokens {
}
pub fn sep_by<T1, T2>(
&mut self,
separator: Token,
separator: &Token,
parser1: Parser<T1>,
parser2: Parser<T2>,
) -> ParseResult<(T1, T2)> {
if let Some(t1) = parser1(self)? {
let _sep = self.next_if(separator).ok_or(Error::UnexpectedEndOfInput)?;
let _sep = self
.next_if(separator)
.ok_or(Error::UnexpectedEndOfInput("between".into()))?;
if let Some(t2) = parser2(self)? {
Ok(Some((t1, t2)))
} else {
@ -97,6 +105,24 @@ impl Tokens {
Ok(None)
}
}
pub fn many_sep_by<T>(&mut self, separator: &Token, parser: Parser<T>) -> ParseResult<Vec<T>> {
if let Some(first) = parser(self)? {
let mut results = vec![first];
while let Some(_) = self.next_if(&separator) {
if let Some(result) = parser(self)? {
results.push(result);
} else {
return Err(Error::UnexpectedTokenForParser(
"sep_by_many".into(),
self.next(),
));
}
}
Ok(Some(results))
} else {
Ok(None)
}
}
pub fn many<T>(&mut self, parser: Parser<T>) -> ParseResult<Vec<T>> {
let mut results = vec![];
while let Some(result) = parser(self)? {
@ -115,6 +141,6 @@ pub enum Error {
expected: Token,
got: Option<LocatedToken>,
},
UnexpectedEndOfInput,
UnexpectedEndOfInput(String),
UnexpectedTokenForParser(String, Option<LocatedToken>),
}