diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 1bd01f7..850d0a6 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -90,59 +90,57 @@ impl Token { let identifier = regex::Regex::new(r#"[A-Za-z_][A-Za-z0-9_']*"#).map_err(|e| TokenizeError::Regex(e))?; let number = regex::Regex::new(r#"([0-9]+\.?[0-9]*)|(\.[0-9])"#).map_err(|e| TokenizeError::Regex(e))?; - if string.is_match(s) { - Ok(Token::Constant(Value::String(s[1..s.len() - 1].to_string()))) - } else if identifier.is_match(s) { - Ok(Token::Identifier(s.to_string())) - } else if number.is_match(s) { - if let Ok(int) = s.parse::() { - Ok(Token::Constant(Value::Int(int))) - } else if let Ok(float) = s.parse::() { - Ok(Token::Constant(Value::Float(float))) - } else { - Err(TokenizeError::InvalidNumericConstant(s.to_string())) - } - } else { - match s { - // First check if s is an operator - "+" => Ok(Token::Operator(Op::Add)), - "-" => Ok(Token::Operator(Op::Sub)), - "*" => Ok(Token::Operator(Op::Mul)), - "/" => Ok(Token::Operator(Op::Div)), - "**" => Ok(Token::Operator(Op::Exp)), - "%" => Ok(Token::Operator(Op::Mod)), - "=" => Ok(Token::Operator(Op::Equ)), - "." => Ok(Token::Operator(Op::LazyEqu)), - "~" => Ok(Token::Operator(Op::Compose)), - "," => Ok(Token::Operator(Op::Id)), - "?" => Ok(Token::Operator(Op::If)), - "??" => Ok(Token::Operator(Op::IfElse)), - ">" => Ok(Token::Operator(Op::GreaterThan)), - "<" => Ok(Token::Operator(Op::LessThan)), - ">=" => Ok(Token::Operator(Op::GreaterThanOrEqualTo)), - "<=" => Ok(Token::Operator(Op::LessThanOrEqualTo)), - "==" => Ok(Token::Operator(Op::EqualTo)), - - // then some keywords - "true" => Ok(Token::Constant(Value::Bool(true))), - "false" => Ok(Token::Constant(Value::Bool(false))), - "not" => Ok(Token::Operator(Op::Not)), - - // Type casting - "int" => Ok(Token::Operator(Op::IntCast)), - "float" => Ok(Token::Operator(Op::FloatCast)), - "bool" => Ok(Token::Operator(Op::BoolCast)), - "string" => Ok(Token::Operator(Op::StringCast)), - - // then variable length keywords - _ => { - if s.starts_with(":") { - Ok(Token::Operator(Op::FunctionDeclare( - get_dot_count(s).map(|x| x - 1).ok_or(TokenizeError::InvalidDynamicOperator(s.to_string()))? - ))) + match s { + // First check if s is an operator + "+" => Ok(Token::Operator(Op::Add)), + "-" => Ok(Token::Operator(Op::Sub)), + "*" => Ok(Token::Operator(Op::Mul)), + "/" => Ok(Token::Operator(Op::Div)), + "**" => Ok(Token::Operator(Op::Exp)), + "%" => Ok(Token::Operator(Op::Mod)), + "=" => Ok(Token::Operator(Op::Equ)), + "." => Ok(Token::Operator(Op::LazyEqu)), + "~" => Ok(Token::Operator(Op::Compose)), + "," => Ok(Token::Operator(Op::Id)), + "?" => Ok(Token::Operator(Op::If)), + "??" => Ok(Token::Operator(Op::IfElse)), + ">" => Ok(Token::Operator(Op::GreaterThan)), + "<" => Ok(Token::Operator(Op::LessThan)), + ">=" => Ok(Token::Operator(Op::GreaterThanOrEqualTo)), + "<=" => Ok(Token::Operator(Op::LessThanOrEqualTo)), + "==" => Ok(Token::Operator(Op::EqualTo)), + + // then some keywords + "true" => Ok(Token::Constant(Value::Bool(true))), + "false" => Ok(Token::Constant(Value::Bool(false))), + "not" => Ok(Token::Operator(Op::Not)), + + // Type casting + "int" => Ok(Token::Operator(Op::IntCast)), + "float" => Ok(Token::Operator(Op::FloatCast)), + "bool" => Ok(Token::Operator(Op::BoolCast)), + "string" => Ok(Token::Operator(Op::StringCast)), + + // then variable length keywords + _ => { + if s.starts_with(":") { + Ok(Token::Operator(Op::FunctionDeclare( + get_dot_count(s).map(|x| x - 1).ok_or(TokenizeError::InvalidDynamicOperator(s.to_string()))? + ))) + } else if string.is_match(s) { + Ok(Token::Constant(Value::String(s[1..s.len() - 1].to_string()))) + } else if identifier.is_match(s) { + Ok(Token::Identifier(s.to_string())) + } else if number.is_match(s) { + if let Ok(int) = s.parse::() { + Ok(Token::Constant(Value::Int(int))) + } else if let Ok(float) = s.parse::() { + Ok(Token::Constant(Value::Float(float))) } else { - Err(TokenizeError::UnableToMatchToken(s.to_string())) + Err(TokenizeError::InvalidNumericConstant(s.to_string())) } + } else { + Err(TokenizeError::UnableToMatchToken(s.to_string())) } } } @@ -258,20 +256,4 @@ impl std::iter::Iterator for Tokenizer { Err(e) => Some(Err(TokenizeError::IO(e))), } } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::str::FromStr; - - #[test] - fn tokenizer() { - let program = ": function x ** x 2 function 1200"; - - let tok = Tokenizer::from_str(program).unwrap(); - let tokens: Vec = tok.collect::>().expect("tokenizer error"); - - println!("{tokens:?}"); - } -} +} \ No newline at end of file