umx_compiler

commit 5585c042a1ee9324d5839fd0d41f828d7358947a
parent aa71b3b112ee508812b5ec1328cef8a8602b58ab
Author: bsandro <email@bsandro.tech>
Date:   Sat, 11 Jun 2022 10:28:31 +0300

Lexer additions, double-char literals support through peeking

Diffstat:
M
lexer/lexer.go
 | 
32
+++++++++++++++++++++++++++++++-
M
lexer/lexer_test.go
 | 
51
+++++++++++++++++++++++++++++++++++++++++++++++++++
M
token/token.go
 | 
18
++++++++++++++++++

3 files changed, 100 insertions(+), 1 deletion(-)
diff --git a/lexer/lexer.go b/lexer/lexer.go
@@ -25,12 +25,25 @@ func (l *Lexer) readChar() {
 	l.readPosition++
 }
 
+func (l *Lexer) peekChar() byte {
+	if l.readPosition < len(l.input) {
+		return l.input[l.readPosition]
+	} else {
+		return 0
+	}
+}
+
 func (l *Lexer) NextToken() token.Token {
 	var tok token.Token
 	l.skipWhitespace()
 	switch l.ch {
 	case '=':
-		tok = newToken(token.ASSIGN, l.ch)
+		if l.peekChar() == '=' {
+			l.readChar()
+			tok = token.Token{Type: token.EQUAL, Literal: "=="}
+		} else {
+			tok = newToken(token.ASSIGN, l.ch)
+		}
 	case ';':
 		tok = newToken(token.SEMICOLON, l.ch)
 	case '(':
@@ -43,6 +56,23 @@ func (l *Lexer) NextToken() token.Token {
 		tok = newToken(token.RCURLY, l.ch)
 	case '+':
 		tok = newToken(token.PLUS, l.ch)
+	case '-':
+		tok = newToken(token.MINUS, l.ch)
+	case '!':
+		if l.peekChar() == '=' {
+			l.readChar()
+			tok = token.Token{Type: token.NOT_EQUAL, Literal: "!="}
+		} else {
+			tok = newToken(token.SHRIEK, l.ch)
+		}
+	case '*':
+		tok = newToken(token.ASTERISK, l.ch)
+	case '/':
+		tok = newToken(token.SLASH, l.ch)
+	case '<':
+		tok = newToken(token.LT, l.ch)
+	case '>':
+		tok = newToken(token.GT, l.ch)
 	case ',':
 		tok = newToken(token.COMMA, l.ch)
 	case 0:
diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go
@@ -12,6 +12,16 @@ let add = fn(x, y) {
   x + y;
 };
 let result = add(five, ten);
+!-*/5;
+6 < 10 > 8;
+if (5 < 10) {
+  return true;
+} else {
+  return false;
+}
+
+10 == 10;
+10 != 8;
 `
 
 	tests := []struct{
@@ -54,6 +64,47 @@ let result = add(five, ten);
 		{token.IDENT, "ten"},
 		{token.RPAREN, ")"},
 		{token.SEMICOLON, ";"},
+
+		{token.SHRIEK, "!"},
+		{token.MINUS, "-"},
+		{token.ASTERISK, "*"},
+		{token.SLASH, "/"},
+		{token.INT, "5"},
+		{token.SEMICOLON, ";"},
+		{token.INT, "6"},
+		{token.LT, "<"},
+		{token.INT, "10"},
+		{token.GT, ">"},
+		{token.INT, "8"},
+		{token.SEMICOLON, ";"},
+
+		{token.IF, "if"},
+		{token.LPAREN, "("},
+		{token.INT, "5"},
+		{token.LT, "<"},
+		{token.INT, "10"},
+		{token.RPAREN, ")"},
+		{token.LCURLY, "{"},
+		{token.RETURN, "return"},
+		{token.TRUE, "true"},
+		{token.SEMICOLON, ";"},
+		{token.RCURLY, "}"},
+		{token.ELSE, "else"},
+		{token.LCURLY, "{"},
+		{token.RETURN, "return"},
+		{token.FALSE, "false"},
+		{token.SEMICOLON, ";"},
+		{token.RCURLY, "}"},
+
+		{token.INT, "10"},
+		{token.EQUAL, "=="},
+		{token.INT, "10"},
+		{token.SEMICOLON, ";"},
+		{token.INT, "10"},
+		{token.NOT_EQUAL, "!="},
+		{token.INT, "8"},
+		{token.SEMICOLON, ";"},
+
 		{token.EOF, ""},
 	}
 
diff --git a/token/token.go b/token/token.go
@@ -14,6 +14,12 @@ const (
 	INT       = "INT"
 	ASSIGN    = "="
 	PLUS      = "+"
+	MINUS     = "-"
+	ASTERISK  = "*"
+	SLASH     = "/"
+	SHRIEK    = "!"
+	LT        = "<"
+	GT        = ">"
 	COMMA     = ","
 	SEMICOLON = ";"
 	LPAREN    = "("
@@ -22,11 +28,23 @@ const (
 	RCURLY    = "}"
 	FUNCTION  = "FUNCTION"
 	LET       = "LET"
+	TRUE      = "TRUE"
+	FALSE     = "FALSE"
+	IF        = "IF"
+	ELSE      = "ELSE"
+	RETURN    = "RETURN"
+	EQUAL     = "=="
+	NOT_EQUAL = "!="
 )
 
 var keywords = map[string]TokenType{
 	"fn": FUNCTION,
 	"let": LET,
+	"true": TRUE,
+	"false": FALSE,
+	"if": IF,
+	"else": ELSE,
+	"return": RETURN,
 }
 
 func LookupIdentifier(ident string) TokenType {