Membuat Bahasa Pemrograman Sederhana dengan Rust dan LLVM - Bagian 1 Lexer

#[derive(Debug, PartialEq, Clone)]
pub enum Token {
	LParen,
	RParen,
	Add,
	Sub,
	Mul,
	Div,
	Num(f64),
	EOF,
	ILLEGAL,
	ASSIGN,
	IDENTIFIER(String),
	If,
	Then,
	Else,
	EQ,
	LT,
	GT,
}

impl From<i32> for Token {
	fn from(n: i32) -> Self {
		Token::Num(n as f64)
	}
}

impl Token {
	//Left Binding Power
	pub fn lbp(&self) -> usize {
		match *self {
			Token::Add => 10,
			Token::Sub => 10,
			Token::Mul => 20,
			Token::Div => 20,
			Token::LParen => 99,
			Token::ASSIGN => 100,
			Token::RParen => 0,
			_ => 0,
		}
	}
}

#[cfg(test)]
mod test {
	use super::*;

	#[test]
	fn test_token_add() {
		let actual = Token::Add.lbp();
		let expected: usize = 10;

		assert_eq!(expected, actual);
	}

	#[test]
	fn test_token_sub() {
		let actual = Token::Sub.lbp();
		let expected: usize = 10;

		assert_eq!(expected, actual);
	}

	#[test]
	fn test_token_mul() {
		let actual = Token::Mul.lbp();
		let expected: usize = 20;

		assert_eq!(expected, actual);
	}

	#[test]
	fn test_token_div() {
		let actual = Token::Div.lbp();
		let expected: usize = 20;

		assert_eq!(expected, actual);
	}

	#[test]
	fn test_token_open_paren() {
		let actual = Token::LParen.lbp();
		let expected: usize = 99;

		assert_eq!(expected, actual);
	}

	#[test]
	fn test_token_close_paren() {
		let actual = Token::RParen.lbp();
		let expected: usize = 0;

		assert_eq!(expected, actual);
	}

	#[test]
	fn test_token_assignment() {
		let actual = Token::ASSIGN.lbp();
		let expected: usize = 100;

		assert_eq!(expected, actual);
	}
}

use std::iter::Peekable;
use std:str:Chars;

use crate::token::Token;

#[derive(Debug)]
pub struct Lexer<'a> {
	input: Peekable<Chars<'a>>,
}

impl <'a> Lexer<'a> {
	pub fn new(input: &str) -> Self {
    	Lexer {
        	input: input.chars().peekable(),
        }
        
	pub fn lex(mut self) -> Vec<Token> {
		let mut tokens = vec![];
        loop {
            let current_token = self.next_token();
            if current_token == Token::EOF || current_token == Token::ILLEGAL {
                tokens.push(current_token);
                break;
            } else {
                tokens.push(current_token);
            }
        }

        tokens
    }
}

// masih di dalam impl Lexer
.... {
	pub fn next_token(&mut self) -> Token {
		match self.input.peek() {
			Some(ch) => match ch {
				' ' | '\t' => {
					self.input.next();
					self.next_token()
				}
				'\n' => {
					self.input.next();
					self.next_token()
				}
				ch if ch.is_numeric() => self.read_numeric(),
				'+' => {
					self.input.next();
					Token::Add
				}
				'-' => {
					self.input.next();
					Token::Sub
				}
				'*' => {
					self.input.next();
					Token::Mul
				}
				'/' => {
					self.input.next();
					Token::Div
				}
				'(' => {
					self.input.next();
					Token::LParen
				}
				')' => {
					self.input.next();
					Token::RParen
				}
				ch if ch.is_alphabetic() => {
					self.read_identifier()
				}
				'=' => {
					self.input.next();
					match self.input.peek() {
						Some(c) => {
							if *c == '=' {
								self.input.next();
								Token::EQ
							} else {
								self.input.next();
								Token::ASSIGN
							}
						}
						_ => {
							self.next_token()
						}
					}
				}
				'<' => {
					self.input.next();
					Token::LT
				}
				'>' => {
					self.input.next();
					Token::GT
				}
				_ => {
					self.input.next();
					Token::ILLEGAL
				}
			},
			None => Token::EOF,
		}
	}

	fn read_numeric(&mut self) -> Token {
		let mut literal = String::new();

		loop {
			match self.input.peek() {
				Some(&ch) => {
					if ch.is_numeric() || ch == '.' {
						literal.push(ch);
						self.input.next();
					} else {
						break;
					}
				}
				_ => break,
			}
		}

		match literal.parse() {
			Ok(l) => Token::Num(l),
			Err(_e) => Token::ILLEGAL,
		}
	}

	fn read_identifier(&mut self) -> Token {
		let mut literal = String::new();

		loop {
			match self.input.peek() {
				Some(&ch) => {
					if !ch.is_alphabetic() {
						break;
					}
					if ch.is_ascii_whitespace() {
						// self.input.next();
						break;
					};
					literal.push(ch);
					self.input.next();
				},
				_ => break
			}
		};

		match literal.as_str() {
			"if" => Token::If,
			"then" => Token::Then,
			"else" => Token::Else,
			_ => Token::IDENTIFIER(literal),
		}
	}
}

impl<'a> Lexer<'a> {
	......
}

#[cfg(test)]
mod test {
	use super::*;

	#[test]
	fn test_num() {
		let lexer = Lexer::new(r#"32"#);
		let tokens = lexer.lex();

		let expected = vec![Token::from(32), Token::EOF];
		assert_eq!(expected, tokens);
	}

	#[test]
	fn test_num_float() {
		let lexer = Lexer::new(r#"32.5"#);
		let tokens = lexer.lex();

		let expected = vec![Token::Num(32.5), Token::EOF];
		assert_eq!(expected, tokens);
	}

	#[test]
	fn test_num_whitespace() {
		let lexer = Lexer::new(r#"32 2"#);
		let tokens = lexer.lex();

		let expected = vec![Token::from(32), Token::from(2), Token::EOF];

		assert_eq!(expected, tokens);
	}

	#[test]
	fn test_num_operator() {
		let lexer = Lexer::new(r#"-+/*"#);
		let tokens = lexer.lex();

		let expected = vec![
			Token::Sub,
			Token::Add,
			Token::Div,
			Token::Mul,
			Token::EOF,
		];

		assert_eq!(expected, tokens);
	}

	#[test]
	fn test_assignment() {
		let lexer = Lexer::new(r#"a = 123"#);
		let tokens = lexer.lex();

		let expected = vec![
			Token::IDENTIFIER("a".to_string()),
			Token::ASSIGN,
			Token::from(123),
			Token::EOF,
		];

		assert_eq!(expected, tokens);
	}

	#[test]
	fn test_conditional() {
		let lexer = Lexer::new(r#"if 1 < 2 then 1 else 0"#);
		let tokens = lexer.lex();

		let expected = vec![
			Token::If,
			Token::from(1),
			Token::LT,
			Token::from(2),
			Token::Then,
			Token::from(1),
			Token::Else,
			Token::from(0),
			Token::EOF,
		];

		assert_eq!(expected, tokens);
	}
}

use std::io;
use std::io::Write;

mod lexer;
mod token;

use lexer::Lexer;

fn main() {
    loop {
        println!();
        print!("> ");

        io::stdout()
            .flush()
            .expect("Could not read from standard input.");

        let mut input = String::new();

        io::stdin()
            .read_line(&mut input)
            .expect("Could not read from standard input.");

        let lexer = Lexer::new(input.as_str());
        let result = lexer.lex();

        println!("{:?}", result);
    }
}