fix order so keywords get parsed correctly

src/tokenizer.rs

@@ -90,59 +90,57 @@ impl Token {
         let identifier = regex::Regex::new(r#"[A-Za-z_][A-Za-z0-9_']*"#).map_err(|e| TokenizeError::Regex(e))?;
         let number = regex::Regex::new(r#"([0-9]+\.?[0-9]*)|(\.[0-9])"#).map_err(|e| TokenizeError::Regex(e))?;
 
-        if string.is_match(s) {
-            Ok(Token::Constant(Value::String(s[1..s.len() - 1].to_string())))
-        } else if identifier.is_match(s) {
-            Ok(Token::Identifier(s.to_string()))
-        } else if number.is_match(s) {
-            if let Ok(int) = s.parse::<i64>() {
-                Ok(Token::Constant(Value::Int(int)))
-            } else if let Ok(float) = s.parse::<f64>() {
-                Ok(Token::Constant(Value::Float(float)))
-            } else {
-                Err(TokenizeError::InvalidNumericConstant(s.to_string()))
-            }
-        } else {
-            match s {
-                // First check if s is an operator
-                "+"  => Ok(Token::Operator(Op::Add)),
-                "-"  => Ok(Token::Operator(Op::Sub)),
-                "*"  => Ok(Token::Operator(Op::Mul)),
-                "/"  => Ok(Token::Operator(Op::Div)),
-                "**" => Ok(Token::Operator(Op::Exp)),
-                "%"  => Ok(Token::Operator(Op::Mod)),
-                "="  => Ok(Token::Operator(Op::Equ)),
-                "."  => Ok(Token::Operator(Op::LazyEqu)),
-                "~"  => Ok(Token::Operator(Op::Compose)),
-                ","  => Ok(Token::Operator(Op::Id)),
-                "?"  => Ok(Token::Operator(Op::If)),
-                "??" => Ok(Token::Operator(Op::IfElse)),
-                ">"  => Ok(Token::Operator(Op::GreaterThan)),
-                "<"  => Ok(Token::Operator(Op::LessThan)),
-                ">=" => Ok(Token::Operator(Op::GreaterThanOrEqualTo)),
-                "<=" => Ok(Token::Operator(Op::LessThanOrEqualTo)),
-                "==" => Ok(Token::Operator(Op::EqualTo)),
+        match s {
+            // First check if s is an operator
+            "+"  => Ok(Token::Operator(Op::Add)),
+            "-"  => Ok(Token::Operator(Op::Sub)),
+            "*"  => Ok(Token::Operator(Op::Mul)),
+            "/"  => Ok(Token::Operator(Op::Div)),
+            "**" => Ok(Token::Operator(Op::Exp)),
+            "%"  => Ok(Token::Operator(Op::Mod)),
+            "="  => Ok(Token::Operator(Op::Equ)),
+            "."  => Ok(Token::Operator(Op::LazyEqu)),
+            "~"  => Ok(Token::Operator(Op::Compose)),
+            ","  => Ok(Token::Operator(Op::Id)),
+            "?"  => Ok(Token::Operator(Op::If)),
+            "??" => Ok(Token::Operator(Op::IfElse)),
+            ">"  => Ok(Token::Operator(Op::GreaterThan)),
+            "<"  => Ok(Token::Operator(Op::LessThan)),
+            ">=" => Ok(Token::Operator(Op::GreaterThanOrEqualTo)),
+            "<=" => Ok(Token::Operator(Op::LessThanOrEqualTo)),
+            "==" => Ok(Token::Operator(Op::EqualTo)),
 
-                // then some keywords
-                "true"  => Ok(Token::Constant(Value::Bool(true))),
-                "false" => Ok(Token::Constant(Value::Bool(false))),
-                "not"   => Ok(Token::Operator(Op::Not)),
+            // then some keywords
+            "true"  => Ok(Token::Constant(Value::Bool(true))),
+            "false" => Ok(Token::Constant(Value::Bool(false))),
+            "not"   => Ok(Token::Operator(Op::Not)),
 
-                // Type casting
-                "int"    => Ok(Token::Operator(Op::IntCast)),
-                "float"  => Ok(Token::Operator(Op::FloatCast)),
-                "bool"   => Ok(Token::Operator(Op::BoolCast)),
-                "string" => Ok(Token::Operator(Op::StringCast)),
+            // Type casting
+            "int"    => Ok(Token::Operator(Op::IntCast)),
+            "float"  => Ok(Token::Operator(Op::FloatCast)),
+            "bool"   => Ok(Token::Operator(Op::BoolCast)),
+            "string" => Ok(Token::Operator(Op::StringCast)),
 
-                // then variable length keywords
-                _ => {
-                    if s.starts_with(":") {
-                        Ok(Token::Operator(Op::FunctionDeclare(
-                            get_dot_count(s).map(|x| x - 1).ok_or(TokenizeError::InvalidDynamicOperator(s.to_string()))?
-                        )))
+            // then variable length keywords
+            _ => {
+                if s.starts_with(":") {
+                    Ok(Token::Operator(Op::FunctionDeclare(
+                        get_dot_count(s).map(|x| x - 1).ok_or(TokenizeError::InvalidDynamicOperator(s.to_string()))?
+                    )))
+                } else if string.is_match(s) {
+                    Ok(Token::Constant(Value::String(s[1..s.len() - 1].to_string())))
+                } else if identifier.is_match(s) {
+                    Ok(Token::Identifier(s.to_string()))
+                } else if number.is_match(s) {
+                    if let Ok(int) = s.parse::<i64>() {
+                        Ok(Token::Constant(Value::Int(int)))
+                    } else if let Ok(float) = s.parse::<f64>() {
+                        Ok(Token::Constant(Value::Float(float)))
                     } else {
-                        Err(TokenizeError::UnableToMatchToken(s.to_string()))
+                        Err(TokenizeError::InvalidNumericConstant(s.to_string()))
                     }
+                } else {
+                    Err(TokenizeError::UnableToMatchToken(s.to_string()))
                 }
             }
         }
@@ -259,19 +257,3 @@ impl<R: BufRead> std::iter::Iterator for Tokenizer<R> {
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::str::FromStr;
-
-    #[test]
-    fn tokenizer() {
-        let program = ": function x ** x 2 function 1200";
-
-        let tok = Tokenizer::from_str(program).unwrap();
-        let tokens: Vec<Token> = tok.collect::<Result<_, TokenizeError>>().expect("tokenizer error");
-
-        println!("{tokens:?}");
-    }
-}