minneelyyyy/lamm
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix order so keywords get parsed correctly

Browse Source
This commit is contained in:
minneelyyyy
2024-10-14 23:58:30 -04:00
parent a1b76956dc
commit 7e8009617e
1 changed files with 50 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/tokenizer.rs
View File

@@ -90,19 +90,6 @@ impl Token {
let identifier = regex::Regex::new(r#"[A-Za-z_][A-Za-z0-9_']*"#).map_err(|e| TokenizeError::Regex(e))?;
let number = regex::Regex::new(r#"([0-9]+\.?[0-9]*)|(\.[0-9])"#).map_err(|e| TokenizeError::Regex(e))?;
if string.is_match(s) {
Ok(Token::Constant(Value::String(s[1..s.len() - 1].to_string())))
} else if identifier.is_match(s) {
Ok(Token::Identifier(s.to_string()))
} else if number.is_match(s) {
if let Ok(int) = s.parse::<i64>() {
Ok(Token::Constant(Value::Int(int)))
} else if let Ok(float) = s.parse::<f64>() {
Ok(Token::Constant(Value::Float(float)))
} else {
Err(TokenizeError::InvalidNumericConstant(s.to_string()))
}
} else {
match s {
// First check if s is an operator
"+" => Ok(Token::Operator(Op::Add)),
@@ -140,6 +127,18 @@ impl Token {
Ok(Token::Operator(Op::FunctionDeclare(
get_dot_count(s).map(|x| x - 1).ok_or(TokenizeError::InvalidDynamicOperator(s.to_string()))?
)))
} else if string.is_match(s) {
Ok(Token::Constant(Value::String(s[1..s.len() - 1].to_string())))
} else if identifier.is_match(s) {
Ok(Token::Identifier(s.to_string()))
} else if number.is_match(s) {
if let Ok(int) = s.parse::<i64>() {
Ok(Token::Constant(Value::Int(int)))
} else if let Ok(float) = s.parse::<f64>() {
Ok(Token::Constant(Value::Float(float)))
} else {
Err(TokenizeError::InvalidNumericConstant(s.to_string()))
}
} else {
Err(TokenizeError::UnableToMatchToken(s.to_string()))
}
@@ -147,7 +146,6 @@ impl Token {
}
}
}
}
/// Tokenize an input stream of source code for a Parser
pub(crate) struct Tokenizer<R: BufRead> {
@@ -259,19 +257,3 @@ impl<R: BufRead> std::iter::Iterator for Tokenizer<R> {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::str::FromStr;
#[test]
fn tokenizer() {
let program = ": function x ** x 2 function 1200";
let tok = Tokenizer::from_str(program).unwrap();
let tokens: Vec<Token> = tok.collect::<Result<_, TokenizeError>>().expect("tokenizer error");
println!("{tokens:?}");
}
}