massive refactor

Cargo.lock

@@ -122,6 +122,7 @@ dependencies = [
  "dotenv",
  "poise",
  "rand",
+ "regex",
  "tokio",
 ]
 

Cargo.toml

@@ -8,3 +8,4 @@ poise = "0.6.1"
 tokio = { version = "1", features = ["full"] }
 dotenv = "0.15.0"
 rand = "0.8.5"
+regex = "1.11"

src/commands/eval/executor.rs

@@ -0,0 +1,259 @@
+use super::{Value, Type, FunctionDeclaration};
+use super::parser::{ParseTree, ParseError};
+
+use std::collections::HashMap;
+use std::borrow::Cow;
+use std::fmt::Display;
+use std::error::Error;
+
+#[derive(Debug)]
+pub enum RuntimeError {
+    ParseError(ParseError),
+    NoOverloadForTypes,
+    ImmutableError(String),
+    VariableUndefined(String),
+    FunctionUndeclared(String),
+    FunctionUndefined(String),
+    NotAVariable(String),
+}
+
+impl Display for RuntimeError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::ParseError(e) => write!(f, "{e}"),
+            Self::NoOverloadForTypes => write!(f, "No overload of this operator exists for these operands"),
+            Self::ImmutableError(ident) => write!(f, "`{ident}` already exists and cannot be redefined"),
+            Self::VariableUndefined(ident) => write!(f, "variable `{ident}` was not defined"),
+            Self::FunctionUndeclared(ident) => write!(f, "function `{ident}` was not declared"),
+            Self::FunctionUndefined(ident) => write!(f, "function `{ident}` was not defined"),
+            Self::NotAVariable(ident) => write!(f, "`{ident}` is a function but was attempted to be used like a variable"),
+        }
+    }
+}
+
+impl Error for RuntimeError {}
+
+#[derive(Clone)]
+enum Evaluation {
+    // at this point, it's type is set in stone
+    Computed(Value),
+
+    // at this point, it's type is unknown, and may contradict a variable's type
+    // or not match the expected value of the expression, this is a runtime error
+    Uncomputed(Box<ParseTree>),
+}
+
+#[derive(Clone)]
+struct Function {
+    decl: FunctionDeclaration,
+    body: Option<Box<ParseTree>>,
+}
+
+#[derive(Clone)]
+enum Object {
+    Variable(Evaluation),
+    Function(Function),
+}
+
+pub struct Executor<I: Iterator<Item = Result<ParseTree, ParseError>>> {
+    exprs: I,
+    globals: HashMap<String, Object>,
+}
+
+impl<I: Iterator<Item = Result<ParseTree, ParseError>>> Executor<I> {
+    pub fn new(exprs: I) -> Self {
+        Self {
+            exprs,
+            globals: HashMap::new(),
+        }
+    }
+
+    fn exec(
+        &mut self,
+        tree: ParseTree,
+        locals: &mut Cow<HashMap<String, Object>>,
+        in_function: Option<&str>) -> Result<Value, RuntimeError>
+    {
+        match tree {
+            ParseTree::Add(x, y) => (self.exec(*x, locals, in_function)? + self.exec(*y, locals, in_function)?)
+                .ok_or(RuntimeError::NoOverloadForTypes),
+            ParseTree::Sub(x, y) => (self.exec(*x, locals, in_function)? - self.exec(*y, locals, in_function)?)
+                .ok_or(RuntimeError::NoOverloadForTypes),
+            ParseTree::Mul(x, y) => (self.exec(*x, locals, in_function)? * self.exec(*y, locals, in_function)?)
+                .ok_or(RuntimeError::NoOverloadForTypes),
+            ParseTree::Div(x, y) => (self.exec(*x, locals, in_function)? / self.exec(*y, locals, in_function)?)
+                .ok_or(RuntimeError::NoOverloadForTypes),
+            ParseTree::Exp(x, y) => match (self.exec(*x, locals, in_function)?, self.exec(*y, locals, in_function)?) {
+                (Value::Int(x), Value::Int(y)) => Ok(Value::Int(x.pow(y as u32))),
+                (Value::Int(x), Value::Float(y)) => Ok(Value::Float((x as f64).powf(y))),
+                (Value::Float(x), Value::Int(y)) => Ok(Value::Float(x.powf(y as f64))),
+                (Value::Float(x), Value::Float(y)) => Ok(Value::Float(x.powf(y))),
+                _ => Err(RuntimeError::NoOverloadForTypes),
+            },
+            ParseTree::EqualTo(x, y) => match (self.exec(*x, locals, in_function)?, self.exec(*y, locals, in_function)?) {
+                (Value::Int(x), Value::Int(y)) => Ok(Value::Bool(x == y)),
+                (Value::Int(x), Value::Float(y)) => Ok(Value::Bool(x as f64 == y)),
+                (Value::Float(x), Value::Int(y)) => Ok(Value::Bool(x == y as f64)),
+                (Value::Float(x), Value::Float(y)) => Ok(Value::Bool(x == y)),
+                (Value::Bool(x), Value::Bool(y)) => Ok(Value::Bool(x == y)),
+                (Value::String(x), Value::String(y)) => Ok(Value::Bool(x == y)),
+                _ => Err(RuntimeError::NoOverloadForTypes)
+            },
+            ParseTree::GreaterThan(x, y) => match (self.exec(*x, locals, in_function)?, self.exec(*y, locals, in_function)?) {
+                (Value::Int(x), Value::Int(y)) => Ok(Value::Bool(x > y)),
+                (Value::Int(x), Value::Float(y)) => Ok(Value::Bool(x as f64 > y)),
+                (Value::Float(x), Value::Int(y)) => Ok(Value::Bool(x > y as f64)),
+                (Value::Float(x), Value::Float(y)) => Ok(Value::Bool(x > y)),
+                _ => Err(RuntimeError::NoOverloadForTypes)
+            },
+            ParseTree::GreaterThanOrEqualTo(x, y) => match (self.exec(*x, locals, in_function)?, self.exec(*y, locals, in_function)?) {
+                (Value::Int(x), Value::Int(y)) => Ok(Value::Bool(x >= y)),
+                (Value::Int(x), Value::Float(y)) => Ok(Value::Bool(x as f64 >= y)),
+                (Value::Float(x), Value::Int(y)) => Ok(Value::Bool(x >= y as f64)),
+                (Value::Float(x), Value::Float(y)) => Ok(Value::Bool(x >= y)),
+                _ => Err(RuntimeError::NoOverloadForTypes)
+            },
+            ParseTree::LessThan(x, y) => match (self.exec(*x, locals, in_function)?, self.exec(*y, locals, in_function)?) {
+                (Value::Int(x), Value::Int(y)) => Ok(Value::Bool(x < y)),
+                (Value::Int(x), Value::Float(y)) => Ok(Value::Bool((x as f64) < y)),
+                (Value::Float(x), Value::Int(y)) => Ok(Value::Bool(x < y as f64)),
+                (Value::Float(x), Value::Float(y)) => Ok(Value::Bool(x < y)),
+                _ => Err(RuntimeError::NoOverloadForTypes)
+            },
+            ParseTree::LessThanOrEqualTo(x, y) => match (self.exec(*x, locals, in_function)?, self.exec(*y, locals, in_function)?) {
+                (Value::Int(x), Value::Int(y)) => Ok(Value::Bool(x <= y)),
+                (Value::Int(x), Value::Float(y)) => Ok(Value::Bool(x as f64 <= y)),
+                (Value::Float(x), Value::Int(y)) => Ok(Value::Bool(x <= y as f64)),
+                (Value::Float(x), Value::Float(y)) => Ok(Value::Bool(x <= y)),
+                _ => Err(RuntimeError::NoOverloadForTypes)
+            },
+            ParseTree::Not(x) => match self.exec(*x, locals, in_function)? {
+                Value::Bool(x) => Ok(Value::Bool(!x)),
+                _ => Err(RuntimeError::NoOverloadForTypes)
+            },
+            ParseTree::Equ(ident, body, scope) => {
+                if self.globals.contains_key(&ident) || locals.contains_key(&ident) {
+                    Err(RuntimeError::ImmutableError(ident.clone()))
+                } else {
+                    let locals = locals.to_mut();
+                    let value = self.exec(*body, &mut Cow::Borrowed(&locals), in_function)?;
+                    locals.insert(ident.clone(), Object::Variable(Evaluation::Computed(value)));
+
+                    self.exec(*scope, &mut Cow::Borrowed(&locals), in_function)
+                }
+            },
+            ParseTree::LazyEqu(ident, body, scope) => {
+                if self.globals.contains_key(&ident) || locals.contains_key(&ident) {
+                    Err(RuntimeError::ImmutableError(ident.clone()))
+                } else {
+                    let locals = locals.to_mut();
+                    locals.insert(ident.clone(), Object::Variable(Evaluation::Uncomputed(body)));
+
+                    self.exec(*scope, &mut Cow::Borrowed(&locals), in_function)
+                }
+            },
+            ParseTree::GlobalEqu(ident, body) => todo!(),
+            ParseTree::LazyGlobalEqu(ident, body) => todo!(),
+            ParseTree::FunctionDefinition(ident, args, r, body, scope) => {
+                let existing = locals.get(&format!("{}{ident}",
+                in_function.map(|s| format!("{s}:")).unwrap_or("".into())))
+                    .or(locals.get(&ident).or(self.globals.get(&ident))).cloned();
+
+                match existing {
+                    Some(_) => Err(RuntimeError::ImmutableError(ident.clone())),
+                    None => {
+                        let locals = locals.to_mut();
+
+                        locals.insert(ident.clone(), Object::Function(Function {
+                            decl: FunctionDeclaration { name: ident.clone(), r, args },
+                            body: Some(body)
+                        }));
+
+                        self.exec(*scope, &mut Cow::Borrowed(&locals), in_function)
+                    }
+                }
+            },
+            ParseTree::Compose(x, y) => {
+                self.exec(*x, locals, in_function)?;
+                self.exec(*y, locals, in_function)
+            },
+            ParseTree::Id(x) => self.exec(*x, locals, in_function),
+            ParseTree::If(cond, body) => if match self.exec(*cond, locals, in_function)? {
+                    Value::Float(f) => f != 0.0,
+                    Value::Int(i) => i != 0,
+                    Value::Bool(b) => b,
+                    Value::String(s) => !s.is_empty(),
+                    Value::Nil => false,
+                } {
+                    self.exec(*body, locals, in_function)
+                } else {
+                    Ok(Value::Nil)
+                },
+            ParseTree::IfElse(cond, istrue, isfalse) => if match self.exec(*cond, locals, in_function)? {
+                Value::Float(f) => f != 0.0,
+                Value::Int(i) => i != 0,
+                Value::Bool(b) => b,
+                Value::String(s) => !s.is_empty(),
+                Value::Nil => false,
+            } {
+                self.exec(*istrue, locals, in_function)
+            } else {
+                self.exec(*isfalse, locals, in_function)
+            },
+            ParseTree::FunctionCall(ident, args) => {
+                let obj = locals.get(&format!("{}{ident}", in_function.unwrap_or("")))
+                    .or(locals.get(&ident)
+                    .or(self.globals.get(&ident))).cloned();
+
+                if let Some(Object::Function(f)) = obj {
+                    let locals = locals.to_mut();
+                    let body = f.body.ok_or(RuntimeError::FunctionUndefined(ident.clone()))?;
+
+                    for ((name, _), tree) in std::iter::zip(f.decl.args, args) {
+                        locals.insert(name.clone(), Object::Variable(Evaluation::Uncomputed(Box::new(tree))));
+                    }
+
+                    self.exec(*body, &mut Cow::Borrowed(&locals), Some(&ident))
+                } else {
+                    Err(RuntimeError::FunctionUndeclared(ident.clone()))
+                }
+            },
+            ParseTree::Variable(ident) => {
+                let locals = locals.to_mut();
+
+                let obj = locals.get(&format!("{}{ident}",
+                    in_function.map(|s| format!("{s}:")).unwrap_or("".into())))
+                        .or(locals.get(&ident).or(self.globals.get(&ident))).cloned();
+
+                if let Some(Object::Variable(eval)) = obj {
+                    match eval {
+                        Evaluation::Computed(v) => Ok(v),
+                        Evaluation::Uncomputed(tree) => {
+                            let v = self.exec(*tree, &mut Cow::Borrowed(&locals), in_function)?;
+                            locals.insert(ident, Object::Variable(Evaluation::Computed(v.clone())));
+
+                            Ok(v)
+                        }
+                    }
+                } else {
+                    Err(RuntimeError::VariableUndefined(ident.clone()))
+                }
+            },
+            ParseTree::Constant(value) => Ok(value),
+        }
+    }
+}
+
+impl<I: Iterator<Item = Result<ParseTree, ParseError>>> Iterator for Executor<I> {
+    type Item = Result<Value, RuntimeError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let expr = self.exprs.next();
+
+        match expr {
+            Some(Ok(expr)) => Some(self.exec(expr, &mut Cow::Borrowed(&HashMap::new()), None)),
+            Some(Err(e)) => Some(Err(RuntimeError::ParseError(e))),
+            None => None,
+        }
+    }
+}

src/commands/eval/mod.rs

@@ -1,27 +1,11 @@
 use crate::common::{Context, Error};
 
-use std::collections::HashMap;
+use std::str::FromStr;
-use std::borrow::Cow;
+use std::fmt::Display;
 
 mod tokenizer;
-mod parse;
+mod parser;
-
+mod executor;
-fn evaluate(expr: &str) -> Result<f64, Error> {
-    let tokens = tokenizer::Token::tokenize(expr)?;
-    let mut tokens = tokens.iter();
-
-    let globals = HashMap::new();
-    let locals = HashMap::new();
-    let mut locals = Cow::Borrowed(&locals);
-
-    let tree = parse::ParseTree::parse(&mut tokens, &globals, &mut locals)?;
-
-    let mut globals = HashMap::new();
-    let locals = HashMap::new();
-    let mut locals = Cow::Borrowed(&locals);
-
-    tree.evaluate(&mut globals, &mut locals)
-}
 
 /// Evaluates an expression (uses Polish Notation)
 #[poise::command(slash_command, prefix_command)]
@@ -29,6 +13,164 @@ pub async fn eval(ctx: Context<'_>,
                   #[rest]
                   expr: String) -> Result<(), Error>
 {
-    ctx.reply(format!("{}", evaluate(&expr)?)).await?;
+    let expr = expr.strip_prefix("```").and_then(|s| s.strip_suffix("```")).unwrap_or(&expr);
+
+    let tok = tokenizer::Tokenizer::from_str(&expr).unwrap(); // Error type is () and never returned
+    let exprs = parser::Parser::new(tok);
+    let exec = executor::Executor::new(exprs);
+
+    let values: Vec<Value> = exec.collect::<Result<_, executor::RuntimeError>>()?;
+
+    let reply: String = values.iter().fold(String::new(), |acc, s| acc + &format!("{s}\n"));
+
+    ctx.reply(reply).await?;
+
     Ok(())
 }
+
+#[derive(Clone, Debug)]
+enum Type {
+    Float,
+    Int,
+    Bool,
+    String,
+    Nil,
+    Any,
+    Function(Box<Type>, Vec<Type>),
+}
+
+#[derive(Clone, Debug)]
+pub enum Value {
+    Float(f64),
+    Int(i64),
+    Bool(bool),
+    String(String),
+    Nil,
+}
+
+impl std::ops::Add for Value {
+    type Output = Option<Value>;
+
+    fn add(self, rhs: Self) -> Self::Output {
+        match self {
+            Self::Int(x) => {
+                match rhs {
+                    Self::Int(y) => Some(Self::Int(x + y)),
+                    Self::Float(y) => Some(Self::Float(x as f64 + y)),
+                    _ => None,
+                }
+            }
+            Self::Float(x) => {
+                match rhs {
+                    Self::Int(y) => Some(Self::Float(x + y as f64)),
+                    Self::Float(y) => Some(Self::Float(x + y)),
+                    _ => None,
+                }
+            }
+            Self::String(x) => {
+                match rhs {
+                    Self::String(y) => Some(Self::String(format!("{x}{y}"))),
+                    _ => None,
+                }
+            }
+            _ => None,
+        }
+    }
+}
+
+impl std::ops::Sub for Value {
+    type Output = Option<Value>;
+
+    fn sub(self, rhs: Self) -> Self::Output {
+        match self {
+            Self::Int(x) => {
+                match rhs {
+                    Self::Int(y) => Some(Self::Int(x - y)),
+                    Self::Float(y) => Some(Self::Float(x as f64 - y)),
+                    _ => None,
+                }
+            }
+            Self::Float(x) => {
+                match rhs {
+                    Self::Int(y) => Some(Self::Float(x - y as f64)),
+                    Self::Float(y) => Some(Self::Float(x - y)),
+                    _ => None,
+                }
+            }
+            _ => None,
+        }
+    }
+}
+
+impl std::ops::Mul for Value {
+    type Output = Option<Value>;
+
+    fn mul(self, rhs: Self) -> Self::Output {
+        match self {
+            Self::Int(x) => {
+                match rhs {
+                    Self::Int(y) => Some(Self::Int(x * y)),
+                    Self::Float(y) => Some(Self::Float(x as f64 * y)),
+                    _ => None,
+                }
+            }
+            Self::Float(x) => {
+                match rhs {
+                    Self::Int(y) => Some(Self::Float(x * y as f64)),
+                    Self::Float(y) => Some(Self::Float(x * y)),
+                    _ => None,
+                }
+            }
+            Self::String(x) => {
+                match rhs {
+                    Self::Int(y) => Some(Self::String(x.repeat(y as usize))),
+                    _ => None,
+                }
+            }
+            _ => None,
+        }
+    }
+}
+
+impl std::ops::Div for Value {
+    type Output = Option<Value>;
+
+    fn div(self, rhs: Self) -> Self::Output {
+        match self {
+            Self::Int(x) => {
+                match rhs {
+                    Self::Int(y) => Some(Self::Int(x / y)),
+                    Self::Float(y) => Some(Self::Float(x as f64 / y)),
+                    _ => None,
+                }
+            }
+            Self::Float(x) => {
+                match rhs {
+                    Self::Int(y) => Some(Self::Float(x / y as f64)),
+                    Self::Float(y) => Some(Self::Float(x / y)),
+                    _ => None,
+                }
+            }
+            _ => None,
+        }
+    }
+}
+
+impl Display for Value {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Float(x) => write!(f, "{x}"),
+            Self::Int(x) => write!(f, "{x}"),
+            Self::Bool(x) => write!(f, "{}", if *x { "true" } else { "false" }),
+            Self::String(x) => write!(f, "{x}"),
+            Self::Nil => write!(f, "nil"),
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct FunctionDeclaration {
+    name: String,
+    r: Type,
+    args: Vec<(String, Type)>,
+}

src/commands/eval/parse.rs

@@ -1,289 +0,0 @@
-use std::error::Error;
-use std::borrow::Cow;
-use std::collections::HashMap;
-use std::fmt::Display;
-use crate::common;
-use super::tokenizer::{self, Token};
-
-#[derive(Clone, Debug)]
-pub enum ParseTree<'a> {
-    Add(Box<ParseTree<'a>>, Box<ParseTree<'a>>),
-    Sub(Box<ParseTree<'a>>, Box<ParseTree<'a>>),
-    Mul(Box<ParseTree<'a>>, Box<ParseTree<'a>>),
-    Div(Box<ParseTree<'a>>, Box<ParseTree<'a>>),
-    Exp(Box<ParseTree<'a>>, Box<ParseTree<'a>>),
-    Equ(&'a str, Box<ParseTree<'a>>, Box<ParseTree<'a>>),
-    GlobalEqu(&'a str, Box<ParseTree<'a>>),
-    Compose(Box<ParseTree<'a>>, Box<ParseTree<'a>>),
-    Id(Box<ParseTree<'a>>),
-    FunctionDeclaration(&'a str, Vec<Object<'a>>, Box<ParseTree<'a>>, Box<ParseTree<'a>>),
-    FunctionApplication(&'a str, Vec<ParseTree<'a>>),
-    Variable(&'a str),
-    Scalar(f64),
-}
-
-#[derive(Clone, Debug)]
-pub struct FunctionDeclaration<'a> {
-    name: &'a str,
-    args: Vec<Object<'a>>,
-}
-
-#[derive(Clone, Debug)]
-pub struct Function<'a> {
-    decl: FunctionDeclaration<'a>,
-    body: Option<Box<ParseTree<'a>>>, // may be used in declarations where a value isn't provided
-}
-
-#[derive(Clone, Debug)]
-pub struct Variable<'a> {
-    name: &'a str,
-    body: Option<Box<ParseTree<'a>>>, // may be used in declarations where a value isn't provided
-}
-
-impl<'a> Variable<'a> {
-    pub fn new(name: &'a str, body: Option<Box<ParseTree<'a>>>) -> Self {
-        Self { name, body }
-    }
-}
-
-#[derive(Clone, Debug)]
-pub enum Object<'a> {
-    Variable(Variable<'a>),
-    Func(Function<'a>),
-}
-
-#[derive(Debug, Clone)]
-pub enum ParseError {
-    UnexpectedEndInput,
-    IdentifierUndefined(String),
-    InvalidIdentifier,
-    FunctionUndefined,
-    VariableUndefined,
-}
-
-impl Display for ParseError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            ParseError::UnexpectedEndInput => write!(f, "Input ended unexpectedly"),
-            ParseError::IdentifierUndefined(name) => write!(f, "Undefined variable `{}`", name),
-            ParseError::InvalidIdentifier => write!(f, "Invalid identifier"),
-            ParseError::FunctionUndefined => write!(f, "Undefined function"),
-            ParseError::VariableUndefined => write!(f, "Undefined variable"),
-        }
-    }
-}
-
-impl Error for ParseError {}
-
-impl<'a> ParseTree<'a> {
-    pub fn parse<I: Iterator<Item = &'a Token>>(
-        tokens: &mut I,
-        globals: &HashMap<String, FunctionDeclaration<'a>>,
-        locals: &mut Cow<HashMap<String, FunctionDeclaration<'a>>>) -> Result<Self, ParseError>
-    {
-        if let Some(token) = tokens.next() {
-            match token {
-                // Just return scalars
-                Token::Scalar(x) => Ok(ParseTree::Scalar(*x)),
-
-                // Get any identifiers as objects from local first then global scope
-                Token::Identifier(ident) => {
-                    // If it is found to be a function, get its argument count.
-                    // During parsing, we only keep track of function definitions
-                    // so that we know how many arguments it takes
-                    if let Some(decl) = locals.clone().get(ident).or(globals.get(ident)) {
-                        let args = decl.args.iter()
-                            .map(|_| ParseTree::parse(tokens, globals, locals)).collect::<Result<Vec<_>, ParseError>>()?;
-
-                        Ok(ParseTree::FunctionApplication(ident, args))
-                    } else {
-                        Ok(ParseTree::Variable(ident))
-                    }
-                }
-
-                Token::Operator(op) => {
-                    match op {
-                        tokenizer::Op::Add => Ok(ParseTree::Add(
-                            Box::new(ParseTree::parse(tokens, globals, locals)?),
-                            Box::new(ParseTree::parse(tokens, globals, locals)?)
-                        )),
-                        tokenizer::Op::Sub => Ok(ParseTree::Sub(
-                            Box::new(ParseTree::parse(tokens, globals, locals)?),
-                            Box::new(ParseTree::parse(tokens, globals, locals)?)
-                        )),
-                        tokenizer::Op::Mul => Ok(ParseTree::Mul(
-                            Box::new(ParseTree::parse(tokens, globals, locals)?),
-                            Box::new(ParseTree::parse(tokens, globals, locals)?)
-                        )),
-                        tokenizer::Op::Div => Ok(ParseTree::Div(
-                            Box::new(ParseTree::parse(tokens, globals, locals)?),
-                            Box::new(ParseTree::parse(tokens, globals, locals)?)
-                        )),
-                        tokenizer::Op::Exp => Ok(ParseTree::Exp(
-                            Box::new(ParseTree::parse(tokens, globals, locals)?),
-                            Box::new(ParseTree::parse(tokens, globals, locals)?)
-                        )),
-                        tokenizer::Op::Equ | tokenizer::Op::LazyEqu => {
-                            let token = tokens.next().ok_or(ParseError::UnexpectedEndInput)?;
-
-                            if let Token::Identifier(ident) = token {
-                                Ok(ParseTree::Equ(
-                                    ident,
-                                    Box::new(ParseTree::parse(tokens, globals, locals)?),
-                                    Box::new(ParseTree::parse(tokens, globals, locals)?)))
-                            } else {
-                                Err(ParseError::InvalidIdentifier)
-                            }
-                        }
-                        tokenizer::Op::GlobalEqu | tokenizer::Op::LazyGlobalEqu => {
-                            let token = tokens.next().ok_or(ParseError::UnexpectedEndInput)?;
-
-                            if let Token::Identifier(ident) = token {
-                                Ok(ParseTree::GlobalEqu(
-                                    ident,
-                                    Box::new(ParseTree::parse(tokens, globals, locals)?)
-                                ))
-                            } else {
-                                Err(ParseError::InvalidIdentifier)
-                            }
-                        }
-                        tokenizer::Op::FunctionDeclare(arg_count) => {
-                            let token = tokens.next().ok_or(ParseError::UnexpectedEndInput)?;
-                            
-                            if let Token::Identifier(ident) = token {
-                                let args: Vec<Object> = tokens.take(*arg_count)
-                                    .map(|token| match token {
-                                        Token::Identifier(s)
-                                            => Ok(Object::Variable(Variable::new(s, None))),
-                                        _ => Err(ParseError::InvalidIdentifier),
-                                    }).collect::<Result<_, ParseError>>()?;
-
-                                if args.len() < *arg_count {
-                                    return Err(ParseError::InvalidIdentifier);
-                                }
-
-                                let locals = locals.to_mut();
-
-                                locals.insert(ident.clone(), FunctionDeclaration {
-                                    name: ident,
-                                    args: args.clone()
-                                });
-
-                                Ok(ParseTree::FunctionDeclaration(
-                                    ident,
-                                    args,
-                                    Box::new(ParseTree::parse(tokens, globals, &mut Cow::Borrowed(&*locals))?),
-                                    Box::new(ParseTree::parse(tokens, globals, &mut Cow::Borrowed(&*locals))?)))
-                            } else {
-                                Err(ParseError::InvalidIdentifier)
-                            }
-                        }
-                        tokenizer::Op::Compose => {
-                            Ok(ParseTree::Compose(
-                                Box::new(ParseTree::parse(tokens, globals, locals)?),
-                                Box::new(ParseTree::parse(tokens, globals, locals)?)
-                            ))
-                        }
-                        tokenizer::Op::Id =>
-                            Ok(ParseTree::Id(Box::new(ParseTree::parse(tokens, globals, locals)?)))
-                    }
-                }
-            }
-        } else {
-            Err(ParseError::UnexpectedEndInput)
-        }
-    }
-
-    pub fn evaluate(
-        self,
-        globals: &mut HashMap<String, Object<'a>>,
-        locals: &mut Cow<HashMap<String, Object<'a>>>) -> Result<f64, common::Error>
-    {
-        match self {
-            ParseTree::Add(l, r) => Ok(l.evaluate(globals, locals)? + r.evaluate(globals, locals)?),
-            ParseTree::Sub(l, r) => Ok(l.evaluate(globals, locals)? - r.evaluate(globals, locals)?),
-            ParseTree::Mul(l, r) => Ok(l.evaluate(globals, locals)? * r.evaluate(globals, locals)?),
-            ParseTree::Div(l, r) => Ok(l.evaluate(globals, locals)? / r.evaluate(globals, locals)?),
-            ParseTree::Exp(l, r)
-                => Ok(l.evaluate(globals, locals)?.powf(r.evaluate(globals, locals)?)),
-            ParseTree::Equ(ident, value, body) => {
-                let value = value.evaluate(globals, locals)?;
-
-                let locals = locals.to_mut();
-
-                locals.insert(ident.to_string(),
-                              Object::Variable(
-                                  Variable::new(ident, Some(Box::new(ParseTree::Scalar(value))))));
-
-                body.evaluate(globals, &mut Cow::Borrowed(&locals))
-            }
-            ParseTree::GlobalEqu(ident, body) => {
-                globals.insert(ident.to_string(),
-                               Object::Variable(Variable::new(ident, Some(body.clone()))));
-
-                Ok(0.0)
-            }
-            ParseTree::Compose(l, r) => {
-                let _ = l.evaluate(globals, locals);
-                r.evaluate(globals, locals)
-            }
-            ParseTree::Id(body) => body.evaluate(globals, locals),
-            ParseTree::FunctionDeclaration(name, args, body, cont) => {
-                let locals = locals.to_mut();
-
-                locals.insert(name.to_string(), Object::Func(Function {
-                    decl: FunctionDeclaration {
-                        name,
-                        args: args.clone(),
-                    },
-                    body: Some(body.clone())
-                }));
-
-                cont.evaluate(globals, &mut Cow::Borrowed(&locals))
-            }
-            ParseTree::FunctionApplication(name, params) => {
-                let locals = locals.to_mut();
-                let obj = locals.get(name).or(globals.get(name)).cloned();
-
-                if let Some(Object::Func(func)) = obj {
-                    for (param, arg) in params.iter().zip(func.decl.args.iter()) {
-                        match arg {
-                            Object::Variable(v)
-                                => locals.insert(
-                                    v.name.to_string(),
-                                    Object::Variable(
-                                        Variable::new(
-                                            &v.name,
-                                            Some(Box::new(
-                                                ParseTree::Scalar(
-                                                    param.clone().evaluate(
-                                                        globals, &mut Cow::Borrowed(&locals))?)))))),
-                            Object::Func(func)
-                                => locals.insert(
-                                    func.decl.name.to_string(),
-                                    Object::Func(func.clone()))
-                        };
-                    }
-
-                    func.body.ok_or(ParseError::FunctionUndefined.into())
-                        .and_then(|body|
-                            body.clone().evaluate(globals, &mut Cow::Borrowed(&locals)))
-                } else {
-                    Err(ParseError::FunctionUndefined.into())
-                }
-            }
-            ParseTree::Variable(ident) => {
-                let locals = locals.to_mut();
-                let obj = locals.get(ident).or(globals.get(ident)).cloned();
-
-                if let Some(Object::Variable(obj)) = obj {
-                    return obj.body.clone().ok_or(ParseError::VariableUndefined.into())
-                        .and_then(|body| body.clone().evaluate(globals, &mut Cow::Borrowed(&locals)));
-                }
-
-                Err(ParseError::IdentifierUndefined(ident.to_string()).into())
-            }
-            ParseTree::Scalar(x) => Ok(x),
-        }
-    }
-}

src/commands/eval/parser.rs

@@ -0,0 +1,288 @@
+use super::{Type, Value, FunctionDeclaration};
+use super::tokenizer::{Token, TokenizeError, Op};
+
+use std::error;
+use std::collections::HashMap;
+use std::fmt::Display;
+use std::borrow::Cow;
+
+#[derive(Debug)]
+pub enum ParseError {
+    NoInput,
+    UnexpectedEndInput,
+    IdentifierUndefined(String),
+    InvalidIdentifier,
+    FunctionUndefined(String),
+    VariableUndefined(String),
+    TokenizeError(TokenizeError),
+}
+
+impl Display for ParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            ParseError::UnexpectedEndInput => write!(f, "Input ended unexpectedly"),
+            ParseError::IdentifierUndefined(name) => write!(f, "Undefined variable `{name}`"),
+            ParseError::InvalidIdentifier => write!(f, "Invalid identifier"),
+            ParseError::FunctionUndefined(name) => write!(f, "Undefined function `{name}`"),
+            ParseError::VariableUndefined(name) => write!(f, "Undefined variable `{name}`"),
+            ParseError::NoInput => write!(f, "No input given"),
+            ParseError::TokenizeError(e) => write!(f, "{e}"),
+        }
+    }
+}
+
+impl error::Error for ParseError {}
+
+#[derive(Clone, Debug)]
+pub enum ParseTree {
+    // Mathematical Operators
+    Add(Box<ParseTree>, Box<ParseTree>),
+    Sub(Box<ParseTree>, Box<ParseTree>),
+    Mul(Box<ParseTree>, Box<ParseTree>),
+    Div(Box<ParseTree>, Box<ParseTree>),
+    Exp(Box<ParseTree>, Box<ParseTree>),
+
+    // Boolean Operations
+    EqualTo(Box<ParseTree>, Box<ParseTree>),
+    GreaterThan(Box<ParseTree>, Box<ParseTree>),
+    GreaterThanOrEqualTo(Box<ParseTree>, Box<ParseTree>),
+    LessThan(Box<ParseTree>, Box<ParseTree>),
+    LessThanOrEqualTo(Box<ParseTree>, Box<ParseTree>),
+    Not(Box<ParseTree>),
+
+    // Defining Objects
+    Equ(String, Box<ParseTree>, Box<ParseTree>),
+    LazyEqu(String, Box<ParseTree>, Box<ParseTree>),
+    GlobalEqu(String, Box<ParseTree>),
+    LazyGlobalEqu(String, Box<ParseTree>),
+    FunctionDefinition(String, Vec<(String, Type)>, Type, Box<ParseTree>, Box<ParseTree>),
+
+    // Functional Operations
+    Compose(Box<ParseTree>, Box<ParseTree>),
+    Id(Box<ParseTree>),
+
+    // Branching
+    If(Box<ParseTree>, Box<ParseTree>),
+    IfElse(Box<ParseTree>, Box<ParseTree>, Box<ParseTree>),
+
+    // Evaluations
+    FunctionCall(String, Vec<ParseTree>),
+    Variable(String),
+    Constant(Value),
+}
+
+impl ParseTree {
+    fn parse<I>(
+        tokens: &mut I,
+        globals: &HashMap<String, FunctionDeclaration>,
+        locals: &mut Cow<HashMap<String, FunctionDeclaration>>) -> Result<Self, ParseError>
+    where
+        I: Iterator<Item = Result<Token, TokenizeError>>,
+    {
+        match tokens.next() {
+            Some(Ok(token)) => {
+                match token {
+                    Token::Constant(c) => Ok(Self::Constant(c)),
+                    Token::Identifier(ident) => {
+                        // If it is found to be a function, get its argument count.
+                        // During parsing, we only keep track of function definitions
+                        // so that we know how many arguments it takes
+                        if let Some(decl) = locals.clone().get(&ident).or(globals.clone().get(&ident)) {
+                            let args = decl.args.iter()
+                                .map(|_| ParseTree::parse(tokens, globals, locals)).collect::<Result<Vec<_>, ParseError>>()?;
+
+                            Ok(ParseTree::FunctionCall(ident.clone(), args))
+                        } else {
+                            Ok(ParseTree::Variable(ident.clone()))
+                        }
+                    }
+                    Token::Operator(op) => {
+                        match op {
+                            Op::Add => Ok(ParseTree::Add(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::Sub => Ok(ParseTree::Sub(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::Mul => Ok(ParseTree::Mul(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::Div => Ok(ParseTree::Div(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::Exp => Ok(ParseTree::Exp(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::Equ | Op::LazyEqu | Op::GlobalEqu | Op::LazyGlobalEqu => {
+                                let token = tokens.next()
+                                    .ok_or(ParseError::UnexpectedEndInput)?
+                                    .map_err(|e| ParseError::TokenizeError(e))?;
+
+                                if let Token::Identifier(ident) = token {
+                                    match op {
+                                        Op::Equ => Ok(ParseTree::Equ(ident.clone(),
+                                            Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                            Box::new(ParseTree::parse(tokens, globals, locals)?)
+                                        )),
+                                        Op::LazyEqu => Ok(ParseTree::LazyEqu(ident.clone(),
+                                            Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                            Box::new(ParseTree::parse(tokens, globals, locals)?)
+                                        )),
+                                        Op::GlobalEqu => Ok(ParseTree::GlobalEqu(ident.clone(),
+                                            Box::new(ParseTree::parse(tokens, globals, locals)?)
+                                        )),
+                                        Op::LazyGlobalEqu => Ok(ParseTree::LazyGlobalEqu(ident.clone(),
+                                            Box::new(ParseTree::parse(tokens, globals, locals)?)
+                                        )),
+                                        _ => panic!("Operator literally changed under your nose"),
+                                    }
+                                } else {
+                                    Err(ParseError::InvalidIdentifier)
+                                }
+                            }
+                            Op::FunctionDeclare(nargs) => {
+                                let token = tokens.next()
+                                    .ok_or(ParseError::UnexpectedEndInput)?
+                                    .map_err(|e| ParseError::TokenizeError(e))?;
+
+                                    if let Token::Identifier(ident) = token {
+                                        let args: Vec<(String, Type)> = tokens.take(nargs)
+                                            .map(|token| match token {
+                                                Ok(Token::Identifier(ident)) => Ok((ident, Type::Any)),
+                                                Ok(_) => Err(ParseError::InvalidIdentifier),
+                                                Err(e) => Err(ParseError::TokenizeError(e)),
+                                            })
+                                            .collect::<Result<Vec<_>, ParseError>>()?;
+
+                                        let locals = locals.to_mut();
+
+                                        locals.insert(ident.clone(), FunctionDeclaration {
+                                            name: ident.clone(),
+                                            r: Type::Any,
+                                            args: args.clone(),
+                                        });
+
+                                        Ok(ParseTree::FunctionDefinition(
+                                            ident,
+                                            args,
+                                            Type::Any,
+                                            Box::new(ParseTree::parse(tokens, globals, &mut Cow::Borrowed(&*locals))?),
+                                            Box::new(ParseTree::parse(tokens, globals, &mut Cow::Borrowed(&*locals))?)))
+                                    } else {
+                                        Err(ParseError::InvalidIdentifier)
+                                    }
+                            }
+                            Op::Compose => Ok(ParseTree::Compose(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::Id => Ok(ParseTree::Id(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::If => Ok(ParseTree::If(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::IfElse => Ok(ParseTree::IfElse(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::EqualTo => Ok(ParseTree::EqualTo(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::GreaterThan => Ok(ParseTree::GreaterThan(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::LessThan => Ok(ParseTree::LessThan(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::GreaterThanOrEqualTo => Ok(ParseTree::GreaterThanOrEqualTo(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::LessThanOrEqualTo => Ok(ParseTree::LessThanOrEqualTo(
+                                Box::new(ParseTree::parse(tokens, globals, locals)?),
+                                Box::new(ParseTree::parse(tokens, globals, locals)?)
+                            )),
+                            Op::Not => Ok(ParseTree::Not(Box::new(ParseTree::parse(tokens, globals, locals)?)))
+                        }
+                    }
+                }
+            },
+            Some(Err(e)) => Err(ParseError::TokenizeError(e)),
+            None => Err(ParseError::NoInput),
+        }
+    }
+}
+
+pub struct Parser<I: Iterator<Item = Result<Token, TokenizeError>>> {
+    tokens: I,
+
+    // These are used to keep track of functions in the current context
+    // by the parser. otherwise the parser would have no way to tell
+    // if the program `* a b 12` is supposed to be ((* a b) (12)) or (* (a b) 12)
+    globals: HashMap<String, FunctionDeclaration>,
+    locals: HashMap<String, FunctionDeclaration>,
+}
+
+impl<I: Iterator<Item = Result<Token, TokenizeError>>> Parser<I> {
+    pub fn new(tokens: I) -> Self {
+        Self {
+            tokens,
+            globals: HashMap::new(),
+            locals: HashMap::new()
+        }
+    }
+
+    pub fn globals(mut self, g: HashMap<String, FunctionDeclaration>) -> Self {
+        self.globals = g;
+        self
+    }
+}
+
+impl<I: Iterator<Item = Result<Token, TokenizeError>>> Iterator for Parser<I> {
+    type Item = Result<ParseTree, ParseError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let tree = ParseTree::parse(&mut self.tokens, &self.globals, &mut Cow::Borrowed(&self.locals));
+
+        match tree {
+            Ok(tree) => Some(Ok(tree)),
+            Err(e) => {
+                match e {
+                    ParseError::NoInput => None,
+                    _ => Some(Err(e)),
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::commands::eval::tokenizer::Tokenizer;
+    use super::{*, ParseTree::*};
+
+    use std::str::FromStr;
+
+    #[test]
+    fn parsing() {
+        let program = "+ 2 2 = pi 3.14 : area r * pi ** r 2 area 16";
+
+        let tokenizer = Tokenizer::from_str(program).expect("couldnt create a paser");
+        let parser = Parser::new(tokenizer);
+
+        for tree in parser {
+            println!("{tree:?}");
+        }
+    }
+}

src/commands/eval/tokenizer.rs

@@ -1,8 +1,36 @@
-use std::error;
+use std::{error, io};
+use std::collections::VecDeque;
 
-use crate::common::Error;
+use super::Value;
 use std::fmt::{Display, Formatter};
-use std::str::FromStr;
+use std::io::{BufRead, Cursor};
+
+#[derive(Debug)]
+pub enum TokenizeError {
+    InvalidDynamicOperator(String),
+    InvalidNumericConstant(String),
+    InvalidIdentifier(String),
+    UnableToMatchToken(String),
+    IO(io::Error),
+}
+
+impl Display for TokenizeError {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            TokenizeError::InvalidDynamicOperator(op)
+                => write!(f, "invalid dynamic operator `{op}`"),
+            TokenizeError::InvalidNumericConstant(t)
+                => write!(f, "invalid numeric constant `{t}`"),
+            TokenizeError::InvalidIdentifier(ident)
+                => write!(f, "invalid identifier `{ident}`"),
+            TokenizeError::UnableToMatchToken(token)
+                => write!(f, "the token `{token}` was unable to be parsed"),
+            TokenizeError::IO(io) => write!(f, "{io}")
+        }
+    }
+}
+
+impl error::Error for TokenizeError {}
 
 #[derive(Debug, Clone)]
 pub enum Op {
@@ -18,87 +46,159 @@ pub enum Op {
     FunctionDeclare(usize),
     Compose,
     Id,
+    If,
+    IfElse,
+    GreaterThan,
+    LessThan,
+    EqualTo,
+    GreaterThanOrEqualTo,
+    LessThanOrEqualTo,
+    Not,
 }
 
 #[derive(Debug, Clone)]
 pub enum Token {
     Identifier(String),
-    Scalar(f64),
     Operator(Op),
+    Constant(Value),
 }
 
-#[derive(Debug, Clone)]
+fn get_dot_count(s: &str) -> Option<usize> {
-struct ParseError(String);
+    s.chars().fold(Some(0), |acc, c|
-
+        match c {
-impl Display for ParseError {
+            ':' => acc.map(|acc| acc + 2),
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            '.' => acc.map(|acc| acc + 1),
-        write!(f, "{}", self.0)
+            _ => None,
-    }
+        }
+    )
 }
 
-impl error::Error for ParseError {
+fn valid_identifier(c: char) -> bool {
-    fn description(&self) -> &str {
+    c.is_alphanumeric() || c == '\'' || c == '_'
-        &self.0
-    }
 }
 
 impl Token {
-    pub fn identifier(str: String) -> Token {
+    fn parse(s: &str) -> Result<Self, TokenizeError> {
-        Token::Identifier(str)
+        let string = regex::Regex::new(r#"".+""#).expect("LOL!");
-    }
 
-    pub fn scalar(value: f64) -> Token {
+        if string.is_match(s) {
-        Token::Scalar(value)
+            return Ok(Token::Constant(Value::String(s[1..s.len() - 1].to_string())));
-    }
+        }
 
-    pub fn operator(op: Op) -> Token {
+        match s {
-        Token::Operator(op)
-    }
-    pub fn tokenize(s: &str) -> Result<Vec<Self>, Error> {
-        s.split_whitespace().map(Token::from_str).collect()
-    }
-}
-
-fn get_dot_count<I: Iterator<Item = char>>(s: I) -> usize {
-    s.fold(0, |acc, c| acc + match c {
-        ':' => 2,
-        '.' => 1,
-        _ => 0,
-    })
-}
-
-impl FromStr for Token {
-    type Err = Error;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        let s = if s.starts_with("\\") { s.chars().skip(1).collect() } else { s.to_string() };
-
-        match s.as_str() {
             // First check if s is an operator
-            "+"  => Ok(Token::operator(Op::Add)),
+            "+"  => Ok(Token::Operator(Op::Add)),
-            "-"  => Ok(Token::operator(Op::Sub)),
+            "-"  => Ok(Token::Operator(Op::Sub)),
-            "*"  => Ok(Token::operator(Op::Mul)),
+            "*"  => Ok(Token::Operator(Op::Mul)),
-            "**" => Ok(Token::operator(Op::Exp)),
+            "**" => Ok(Token::Operator(Op::Exp)),
-            "/"  => Ok(Token::operator(Op::Div)),
+            "/"  => Ok(Token::Operator(Op::Div)),
-            "="  => Ok(Token::operator(Op::Equ)),
+            "="  => Ok(Token::Operator(Op::Equ)),
-            "."  => Ok(Token::operator(Op::LazyEqu)),
+            "."  => Ok(Token::Operator(Op::LazyEqu)),
-            "=>" => Ok(Token::operator(Op::GlobalEqu)),
+            "=>" => Ok(Token::Operator(Op::GlobalEqu)),
-            ".>" => Ok(Token::operator(Op::LazyGlobalEqu)),
+            ".>" => Ok(Token::Operator(Op::LazyGlobalEqu)),
-            "~"  => Ok(Token::operator(Op::Compose)),
+            "~"  => Ok(Token::Operator(Op::Compose)),
-            "," => Ok(Token::operator(Op::Id)),
+            "," => Ok(Token::Operator(Op::Id)),
+            "?" => Ok(Token::Operator(Op::If)),
+            "??" => Ok(Token::Operator(Op::IfElse)),
+            ">" => Ok(Token::Operator(Op::GreaterThan)),
+            "<" => Ok(Token::Operator(Op::LessThan)),
+            ">=" => Ok(Token::Operator(Op::GreaterThanOrEqualTo)),
+            "<=" => Ok(Token::Operator(Op::LessThanOrEqualTo)),
+            "==" => Ok(Token::Operator(Op::EqualTo)),
+
+            // then some keywords
+            "true" => Ok(Token::Constant(Value::Bool(true))),
+            "false" => Ok(Token::Constant(Value::Bool(false))),
+            "not" => Ok(Token::Operator(Op::Not)),
+
+            // then variable length keywords, constants, and identifiers
             _ => {
-                // variable length operators
                 if s.starts_with(':') {
-                    Ok(Token::operator(Op::FunctionDeclare(1 + get_dot_count(s[1..].chars()))))
+                    Ok(Token::Operator(Op::FunctionDeclare(
+                        get_dot_count(s).map(|x| x - 1).ok_or(TokenizeError::InvalidDynamicOperator(s.to_string()))?
+                    )))
                 } else if s.starts_with(|c| char::is_digit(c, 10)) {
-                    Ok(Token::scalar(s.parse()?))
+                    if let Ok(int) = s.parse::<i64>() {
-                } else if s.starts_with(char::is_alphabetic)
+                        Ok(Token::Constant(Value::Int(int)))
-                        && s.chars().skip(1).all(char::is_alphanumeric) {
+                    } else if let Ok(float) = s.parse::<f64>() {
-                    Ok(Token::identifier(s.to_string()))
+                        Ok(Token::Constant(Value::Float(float)))
+                    } else {
+                        Err(TokenizeError::InvalidNumericConstant(s.to_string()))
+                    }
+                } else if s.starts_with(valid_identifier) {
+                    let valid = s.chars().skip(1).all(valid_identifier);
+                    valid.then(|| Token::Identifier(s.to_string())).ok_or(TokenizeError::InvalidIdentifier(s.to_string()))
                 } else {
-                    Err(Box::new(ParseError(format!("Failed to parse \"{}\"", s))))
+                    Err(TokenizeError::UnableToMatchToken(s.to_string()))
                 }
             }
         }
     }
 }
+
+pub struct Tokenizer<R: BufRead> {
+    reader: R,
+    tokens: VecDeque<Token>,
+}
+
+impl<R: BufRead> Tokenizer<R> {
+    pub fn new(reader: R) -> Self {
+        Self {
+            reader,
+            tokens: VecDeque::new(),
+        }
+    }
+}
+
+impl std::str::FromStr for Tokenizer<Cursor<String>> {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let cursor = Cursor::new(s.to_string());
+        Ok(Tokenizer::new(cursor))
+    }
+}
+
+impl<R: BufRead> std::iter::Iterator for Tokenizer<R> {
+    type Item = Result<Token, TokenizeError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(token) = self.tokens.pop_front() {
+            return Some(Ok(token));
+        }
+
+        let mut input = String::new();
+
+        match self.reader.read_to_string(&mut input) {
+            Ok(0) => return None,
+            Err(e) => return Some(Err(TokenizeError::IO(e))),
+            _ => {
+                let re = regex::Regex::new(r#"[a-zA-Z0-9\.'_]+|[`~!@#\$%\^&\*\(\)\+-=\[\]\{\}\\|;:,<\.>/\?]+|("[^"]+")"#).expect("This wont fail promise :3");
+
+                for token in re.find_iter(input.as_str()).map(|mat| mat.as_str()).map(Token::parse) {
+                    match token {
+                        Ok(token) => self.tokens.push_back(token),
+                        Err(e) => return Some(Err(e)),
+                    }
+                }
+
+                self.tokens.pop_front().map(|x| Ok(x))
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::str::FromStr;
+
+    #[test]
+    fn test_string_parsing() {
+        let program = r#"+ "hi" "bye" "whats good""#;
+        let tokenizer = Tokenizer::from_str(program).unwrap();
+        let tokens: Vec<Token> = tokenizer.collect::<Result<_, TokenizeError>>().expect("tokenizer failure");
+
+        println!("{tokens:?}");
+    }
+}