Create lexer.rs

Author: KajizukaTaichi

src/lexer.rs

@@ -0,0 +1,60 @@
+pub fn tokenize(input: &str) -> Option<Vec<String>> {
+    let mut tokens: Vec<String> = Vec::new();
+    let mut current_token = String::new();
+    let mut in_parentheses: usize = 0;
+    let mut in_quote = false;
+    let mut is_escape = false;
+
+    for c in input.chars() {
+        if is_escape {
+            current_token.push(match c {
+                'n' => '\n',
+                't' => '\t',
+                'r' => '\r',
+                _ => c,
+            });
+            is_escape = false;
+        } else {
+            match c {
+                '(' | '{' | '[' if !in_quote => {
+                    current_token.push(c);
+                    in_parentheses += 1;
+                }
+                ')' | '}' | ']' if !in_quote => {
+                    current_token.push(c);
+                    in_parentheses.checked_sub(1).map(|x| in_parentheses = x);
+                }
+                '"' | '\'' | '`' => {
+                    in_quote = !in_quote;
+                    current_token.push(c);
+                }
+                '\\' if in_quote => {
+                    current_token.push(c);
+                    is_escape = true;
+                }
+                other => {
+                    if other.is_whitespace() && !in_quote {
+                        if in_parentheses != 0 {
+                            current_token.push(c);
+                        } else if !current_token.is_empty() {
+                            tokens.push(current_token.clone());
+                            current_token.clear();
+                        }
+                    } else {
+                        current_token.push(c);
+                    }
+                }
+            }
+        }
+    }
+
+    // Syntax error check
+    if is_escape || in_quote || in_parentheses != 0 {
+        return None;
+    }
+    if !current_token.is_empty() {
+        tokens.push(current_token.clone());
+        current_token.clear();
+    }
+    Some(tokens)
+}